def upload(self, dir, datafile, **kwargs):
error = ''
fn = ''
if datafile:
path = Path(op.join(datapath, dir))
if not path:
path.make()
fn = path + datafile.filename
if not fn.is_legal:
error = "'%s' is not legal"
if fn and 'overwrite' not in kwargs:
error = "'%s' exists already, if you want to overwrite the old version, check allow overwrite" % fn.name
# Buffer file for first check encoding and secondly upload file
with BytesIO(datafile.file.read()) as filebuffer:
# determine file encodings
result = chardet.detect(filebuffer.read())
# Reset file buffer
filebuffer.seek(0)
# if chardet can determine file encoding, check it and warn respectively
# otherwise state not detecting
# TODO: chardet cannot determine sufficent amount of encodings, such as utf-16-le
if result['encoding']:
file_encoding = result['encoding'].lower()
# TODO: outsource valid encodings
if not (file_encoding in ['utf-8', 'ascii']
or 'utf-8' in file_encoding):
log.error("WARNING: encoding of file {} is {}".format(
datafile.filename, file_encoding))
else:
msg = "WARNING: encoding of file {} is not detectable".format(
datafile.filename)
log.error(msg)
try:
write_to_file(fn.absolute, filebuffer)
fn.setownergroup()
except:
error += '\n' + traceback()
print(error)
if "uploadimport" in kwargs and not error:
url = '/download/to_db?filename=' + escape(fn.href)
else:
url = '/download?dir=' + escape(dir)
if error:
url += '&error=' + escape(error)
raise web.HTTPRedirect(url)
def removefile(self, dir, filename):
path = Path(op.join(datapath + '/' + dir, filename))
error = ''
if path.exists():
try:
os.remove(path.absolute)
except:
error = "Could not delete the file. A good reason would be a mismatch of user rights on the server " \
"file system"
else:
error = "File not found. Is it already deleted?"
if dir == '.' and error == '':
return self.index()
else:
# TODO: Remove this hack
raise web.HTTPRedirect("/download/?dir=%s&error=%s" % (dir, error))
def saveindex(self, dir, s):
"""Saves the string s to index.html
"""
path = Path(op.join(datapath, dir, 'index.html'))
s = s.replace('\r', '')
f = open(path.absolute, 'wb')
f.write(s)
f.close()
return web.markdown(s)
def logimport(self, filename, kwargs, import_with_class=LogbookImport):
"""
:param filename:
:param kwargs:
:param import_with_class:
:return:
"""
import dataimport.importlog as il
t0 = time.time()
absfile = web.abspath(filename.strip('/'))
path = Path(absfile)
import dataimport as di
error = web.markdown(di.checkimport(path.absolute))
config = None
if import_with_class == ManualMeasurementsImport:
config = ManualMeasurementsImport.from_file(path.absolute)
print("path = %s;\nabsfile = %s" % (path, absfile))
from cherrypy import log
log("Import with class %s" % import_with_class.__name__)
li = import_with_class(absfile, web.user(), config=config)
# TODO: Sometimes this is causing a delay
logs, cancommit = li('commit' in kwargs)
# TODO: REFACTORING FOR MAINTAINABILITY
t1 = time.time()
log("Imported in %.2f s" % (t1 - t0))
if 'commit' in kwargs and cancommit:
di.savetoimports(absfile, web.user(), ["_various_as_its_manual"])
raise web.HTTPRedirect('/download?dir=' + escape(path.up()))
else:
return web.render('logimport.html', filename=path, logs=logs,
cancommit=cancommit, error=error)\
.render('html', doctype='html')
def index(self, dir='', error='', **kwargs):
path = Path(op.join(datapath, dir))
files = []
directories = []
if path.isdir() and path.is_legal:
for fn in path.listdir():
if not fn.startswith('.'):
child = path.child(fn)
if child.isdir():
directories.append(child)
elif child.isfile():
files.append(child)
files.sort()
directories.sort()
else:
error = '%s is not a valid directory' % dir
return web.render('download.html', error=error, files=files,
directories=directories, curdir=path,
max_size=conf.CFG_UPLOAD_MAX_SIZE)\
.render('html', doctype='html')
def getindex(self, dir):
index = Path(op.join(datapath, dir, 'index.html'))
io = StringIO()
if index.exists():
io.write(open(index.absolute).read())
imphist = Path(op.join(datapath, dir, '.import.hist'))
if imphist.exists():
io.write('\n')
for l in open(imphist.absolute):
ls = l.split(',', 3)
io.write(' * file:%s/%s imported by user:%s at %s into %s\n' %
tuple([imphist.up()] + ls))
return web.markdown(io.getvalue())
def uploadzone_upload():
if not current_user.is_authenticated:
return redirect(url_for('login_page', next=url_for('uploadzone_view')))
fl = request.files.get('file')
if fl and fl.filename:
filename = secure_filename(fl.filename)
p = join(cfg.UPLOAD_ZONE_PATH,
get_formatted_datetime('%d-%m-%Y %H-%M-%S'))
if not Path(p).exists():
create_folder(p)
fl.save(join(p, filename))
session['uploadzone_info'] = 'Uploaded Successfully'
return redirect(url_for('uploadzone_view'))
else:
session['uploadzone_error'] = 'Upload Failed. File not found'
return redirect(url_for('uploadzone_view'))
def select_folder(self, multiple=False, drag=False, folders=None):
if multiple:
file_dialog = QtWidgets.QFileDialog(directory=self.last_selected_dir)
file_dialog.setFileMode(QtWidgets.QFileDialog.DirectoryOnly)
file_dialog.setOption(QtWidgets.QFileDialog.DontUseNativeDialog, True)
file_dialog.setWindowTitle("Select Folders")
file_dialog.setWindowIcon(QtGui.QIcon(self.main_icon))
f_tree_view = file_dialog.findChild(QtWidgets.QTreeView)
if f_tree_view:
f_tree_view.setSelectionMode(QtWidgets.QAbstractItemView.MultiSelection)
if not file_dialog.exec():
return
folders = file_dialog.selectedFiles()
elif drag:
folders = [f for f in folders if is_valid_dir(f)]
else:
folder = str(QtWidgets.QFileDialog.getExistingDirectory(self, "Select Directory", directory=self.last_selected_dir))
if folder == '':
return
folder = str(Path(folder))
self.last_selected_dir = join(*folder.split('\\')[:len(folder.split('\\'))-1])
folders = [folder]
self.FILEDATA.select_dirs(folders)
self.ui.txt_stat.setText(self.FILEDATA.get_status_txt())
def newfolder(self, dir, newfolder):
error = ''
if newfolder:
if ' ' in newfolder:
error = "The folder name may not include a space!"
else:
try:
path = Path(op.join(datapath, dir, newfolder))
if not path:
path.make()
path.setownergroup()
else:
error = "Folder %s exists already!" % newfolder
except:
error = traceback()
else:
error = 'Forgotten to give your new folder a name?'
url = '/download?dir=' + escape(dir)
if error:
url += '&error=' + escape(error)
return self.index(dir=dir, error=error)
def start_action_dialog(self):
i1 = self.ui.input_1.currentIndex()
i2 = self.ui.input_2.text()
include_subfolder = self.ui.side_input_1.isChecked()
if i2.strip() == '':
return QtWidgets.QMessageBox.warning(self, "Warning", f"<p style=\"font-size: 10pt;\"><b>Error Found.</b> Please, give Output Directory.</p>")
i2 = Path(i2)
if not (i2.exists() and i2.is_dir()):
return QtWidgets.QMessageBox.warning(self, "Warning", f"<p style=\"font-size: 10pt;\"><b>Error Found.</b> Output Directory not exist.</p>")
i2 = str(i2)
folders = self.FILEDATA.get_all_folders(include_subfolder)
data = {
's2': self.ui.side_input_2.isChecked(),
's3': self.ui.side_input_3.isChecked(),
's4': self.ui.side_input_4.text(),
's5': self.ui.side_input_5.text(),
's6': self.ui.side_input_6.text(),
's7': self.ui.side_input_7.text(),
's8': self.ui.side_input_8.isChecked(),
's9': self.ui.side_input_9.isChecked(),
's10': self.ui.side_input_10.text(),
's11': self.ui.side_input_11.text(),
's12': self.ui.side_input_12.text(),
's13': self.ui.side_input_13.text(),
}
if self.action_dialog == None:
self.action_dialog = QtWidgets.QWidget()
self.action_dialog.setWindowTitle("Action")
self.action_dialog.ui = Ui_ActionDialog()
self.action_dialog.ui.setupUi(self.action_dialog)
self.action_dialog.setFixedHeight(223)
self.action_dialog.setFixedWidth(515)
self.action_dialog.ui.progressBar.setMinimum(0)
self.action_dialog.ui.pushButton_start.clicked.connect(self.go_for_work)
self.action_dialog.ui.pushButton_cancel.clicked.connect(self.action_dialog.close)
self.action_dialog.ui.pushButton_finish.clicked.connect(self.action_dialog.close)
self.action_dialog.ui.pushButton_start.setEnabled(True)
self.action_dialog.ui.pushButton_cancel.setEnabled(True)
self.action_dialog.ui.pushButton_finish.setEnabled(False)
self.action_dialog.ui.label_desDir.setText(get_prefferd_text(i2))
action = 'Copying :'
action_ = 'Copied'
if i1 == 1:
action = 'Moving :'
action_ = 'Moved'
self.action_dialog.ui.label_action_going.setText(action)
self.action_dialog.ui.label_action.setText(action_)
self.action_dialog.ui.stackedWidget.setCurrentIndex(1)
self.action_dialog.ui.label_statBar.setText("""<html><body><p align="center"><span style="font-size:18pt;">Click 'Start' to go</span></p></body></html>""")
self.action_dialog.ui.label_done.setText("0")
self.action_dialog.ui.label_failed.setText("0")
self.action_dialog.ui.label_notAllowed.setText("0")
self.action_dialog.ui.label_ignored.setText("0")
self.action_dialog.ui.progressBar.setValue(0)
self.action_dialog.data_data = data
self.action_dialog.data_folders = folders
self.action_dialog.data_action = i1
self.action_dialog.data_des = i2
self.action_dialog.action_done = 0
self.action_dialog.destroy()
self.action_dialog.show()
def select_output_folder(self):
folder = str(QtWidgets.QFileDialog.getExistingDirectory(self, "Select Output Directory", directory=self.last_selected_out_dir))
if folder != '':
folder = str(Path(folder))
self.last_selected_out_dir = join(*folder.split('\\')[:len(folder.split('\\'))-1])
self.ui.input_2.setText(folder)
def log(msg, stream=sys.stdout):
stream.write(nowstr() + ' ' + msg + '\n')
stream.flush()
if len(sys.argv) != 4:
print("usage: climateimport.py [newfile.dat] [siteid] [archivefile.dat]")
# Get filename for import from cmdline
def nowstr():
return datetime.now().strftime('%Y-%m-%d %H:%M')
path = Path(sys.argv[1])
i = 0
log('Wait for ' + path.absolute)
while not path.exists():
time.sleep(60)
i += 1
if i > 20:
log('After 20 tries does ' + path.basename + ' not exist', sys.stderr)
sys.exit(1)
try:
# Get siteid from cmdline
siteid = int(sys.argv[2])
log('Process settings for ' + path.basename)
# Create a TextImport
def instrumentimport(self, filename, kwargs):
"""
Loads instrument data using a .conf file
"""
t0 = time.time()
# Error streams
errorstream = StringIO()
# TODO: Major refactoring of this code logic, when to load gaps, etc.
path = Path(web.abspath(filename.strip('/')))
print("path = %s" % path)
import dataimport as di
error = web.markdown(di.checkimport(path.absolute))
startdate = kwargs.get('startdate')
enddate = kwargs.get('enddate')
siteid = web.conv(int, kwargs.get('site'))
instrumentid = web.conv(int, kwargs.get('instrument'))
config = di.getconfig(path.absolute)
if not config:
errorstream.write(
"No config available. Please provide a config for"
" computing a decent result.")
if config:
valuetype = [e.valuetype for e in config.columns]
if config:
config.href = Path(config.filename).href
if startdate:
startdate = web.parsedate(startdate)
if enddate:
enddate = web.parsedate(enddate)
stats = gaps = datasets = None
sites = []
possible_datasets = []
if startdate and enddate:
gaps = [(startdate, enddate)]
if siteid and (instrumentid or config):
absfile = web.abspath(filename.strip('/'))
adapter = di.get_adapter(absfile, web.user(), siteid, instrumentid,
startdate, enddate)
adapter.errorstream = errorstream
if 'loadstat' in kwargs:
stats = adapter.get_statistic()
startdate = min(v.start for v in stats.values())
enddate = max(v.end for v in stats.values())
if 'importdb' in kwargs and startdate and enddate:
gaps = None
datasets = di.importfile(absfile, web.user(), siteid,
instrumentid, startdate, enddate)
else:
gaps = di.finddateGaps(siteid, instrumentid, valuetype,
startdate, enddate)
error = adapter.errorstream.getvalue()
adapter.errorstream.close()
t1 = time.time()
log("Imported in %.2f s" % (t1 - t0))
return web.render('dbimport.html', di=di, error=error,
filename=filename, instrumentid=instrumentid,
dirlink=path.up(), siteid=siteid, gaps=gaps,
stats=stats, datasets=datasets, config=config,
#mmimport=isinstance(config, di.mm.ImportManualMeasurementsDescription),
sites=sites, possible_datasets=possible_datasets)\
.render('html', doctype='html')