def upload(self, dir, datafile, **kwargs): error = '' fn = '' if datafile: path = Path(op.join(datapath, dir)) if not path: path.make() fn = path + datafile.filename if not fn.is_legal: error = "'%s' is not legal" if fn and 'overwrite' not in kwargs: error = "'%s' exists already, if you want to overwrite the old version, check allow overwrite" % fn.name # Buffer file for first check encoding and secondly upload file with BytesIO(datafile.file.read()) as filebuffer: # determine file encodings result = chardet.detect(filebuffer.read()) # Reset file buffer filebuffer.seek(0) # if chardet can determine file encoding, check it and warn respectively # otherwise state not detecting # TODO: chardet cannot determine sufficent amount of encodings, such as utf-16-le if result['encoding']: file_encoding = result['encoding'].lower() # TODO: outsource valid encodings if not (file_encoding in ['utf-8', 'ascii'] or 'utf-8' in file_encoding): log.error("WARNING: encoding of file {} is {}".format( datafile.filename, file_encoding)) else: msg = "WARNING: encoding of file {} is not detectable".format( datafile.filename) log.error(msg) try: write_to_file(fn.absolute, filebuffer) fn.setownergroup() except: error += '\n' + traceback() print(error) if "uploadimport" in kwargs and not error: url = '/download/to_db?filename=' + escape(fn.href) else: url = '/download?dir=' + escape(dir) if error: url += '&error=' + escape(error) raise web.HTTPRedirect(url)
def removefile(self, dir, filename): path = Path(op.join(datapath + '/' + dir, filename)) error = '' if path.exists(): try: os.remove(path.absolute) except: error = "Could not delete the file. A good reason would be a mismatch of user rights on the server " \ "file system" else: error = "File not found. Is it already deleted?" if dir == '.' and error == '': return self.index() else: # TODO: Remove this hack raise web.HTTPRedirect("/download/?dir=%s&error=%s" % (dir, error))
def saveindex(self, dir, s): """Saves the string s to index.html """ path = Path(op.join(datapath, dir, 'index.html')) s = s.replace('\r', '') f = open(path.absolute, 'wb') f.write(s) f.close() return web.markdown(s)
def logimport(self, filename, kwargs, import_with_class=LogbookImport): """ :param filename: :param kwargs: :param import_with_class: :return: """ import dataimport.importlog as il t0 = time.time() absfile = web.abspath(filename.strip('/')) path = Path(absfile) import dataimport as di error = web.markdown(di.checkimport(path.absolute)) config = None if import_with_class == ManualMeasurementsImport: config = ManualMeasurementsImport.from_file(path.absolute) print("path = %s;\nabsfile = %s" % (path, absfile)) from cherrypy import log log("Import with class %s" % import_with_class.__name__) li = import_with_class(absfile, web.user(), config=config) # TODO: Sometimes this is causing a delay logs, cancommit = li('commit' in kwargs) # TODO: REFACTORING FOR MAINTAINABILITY t1 = time.time() log("Imported in %.2f s" % (t1 - t0)) if 'commit' in kwargs and cancommit: di.savetoimports(absfile, web.user(), ["_various_as_its_manual"]) raise web.HTTPRedirect('/download?dir=' + escape(path.up())) else: return web.render('logimport.html', filename=path, logs=logs, cancommit=cancommit, error=error)\ .render('html', doctype='html')
def index(self, dir='', error='', **kwargs): path = Path(op.join(datapath, dir)) files = [] directories = [] if path.isdir() and path.is_legal: for fn in path.listdir(): if not fn.startswith('.'): child = path.child(fn) if child.isdir(): directories.append(child) elif child.isfile(): files.append(child) files.sort() directories.sort() else: error = '%s is not a valid directory' % dir return web.render('download.html', error=error, files=files, directories=directories, curdir=path, max_size=conf.CFG_UPLOAD_MAX_SIZE)\ .render('html', doctype='html')
def getindex(self, dir): index = Path(op.join(datapath, dir, 'index.html')) io = StringIO() if index.exists(): io.write(open(index.absolute).read()) imphist = Path(op.join(datapath, dir, '.import.hist')) if imphist.exists(): io.write('\n') for l in open(imphist.absolute): ls = l.split(',', 3) io.write(' * file:%s/%s imported by user:%s at %s into %s\n' % tuple([imphist.up()] + ls)) return web.markdown(io.getvalue())
def uploadzone_upload(): if not current_user.is_authenticated: return redirect(url_for('login_page', next=url_for('uploadzone_view'))) fl = request.files.get('file') if fl and fl.filename: filename = secure_filename(fl.filename) p = join(cfg.UPLOAD_ZONE_PATH, get_formatted_datetime('%d-%m-%Y %H-%M-%S')) if not Path(p).exists(): create_folder(p) fl.save(join(p, filename)) session['uploadzone_info'] = 'Uploaded Successfully' return redirect(url_for('uploadzone_view')) else: session['uploadzone_error'] = 'Upload Failed. File not found' return redirect(url_for('uploadzone_view'))
def select_folder(self, multiple=False, drag=False, folders=None): if multiple: file_dialog = QtWidgets.QFileDialog(directory=self.last_selected_dir) file_dialog.setFileMode(QtWidgets.QFileDialog.DirectoryOnly) file_dialog.setOption(QtWidgets.QFileDialog.DontUseNativeDialog, True) file_dialog.setWindowTitle("Select Folders") file_dialog.setWindowIcon(QtGui.QIcon(self.main_icon)) f_tree_view = file_dialog.findChild(QtWidgets.QTreeView) if f_tree_view: f_tree_view.setSelectionMode(QtWidgets.QAbstractItemView.MultiSelection) if not file_dialog.exec(): return folders = file_dialog.selectedFiles() elif drag: folders = [f for f in folders if is_valid_dir(f)] else: folder = str(QtWidgets.QFileDialog.getExistingDirectory(self, "Select Directory", directory=self.last_selected_dir)) if folder == '': return folder = str(Path(folder)) self.last_selected_dir = join(*folder.split('\\')[:len(folder.split('\\'))-1]) folders = [folder] self.FILEDATA.select_dirs(folders) self.ui.txt_stat.setText(self.FILEDATA.get_status_txt())
def newfolder(self, dir, newfolder): error = '' if newfolder: if ' ' in newfolder: error = "The folder name may not include a space!" else: try: path = Path(op.join(datapath, dir, newfolder)) if not path: path.make() path.setownergroup() else: error = "Folder %s exists already!" % newfolder except: error = traceback() else: error = 'Forgotten to give your new folder a name?' url = '/download?dir=' + escape(dir) if error: url += '&error=' + escape(error) return self.index(dir=dir, error=error)
def start_action_dialog(self): i1 = self.ui.input_1.currentIndex() i2 = self.ui.input_2.text() include_subfolder = self.ui.side_input_1.isChecked() if i2.strip() == '': return QtWidgets.QMessageBox.warning(self, "Warning", f"<p style=\"font-size: 10pt;\"><b>Error Found.</b> Please, give Output Directory.</p>") i2 = Path(i2) if not (i2.exists() and i2.is_dir()): return QtWidgets.QMessageBox.warning(self, "Warning", f"<p style=\"font-size: 10pt;\"><b>Error Found.</b> Output Directory not exist.</p>") i2 = str(i2) folders = self.FILEDATA.get_all_folders(include_subfolder) data = { 's2': self.ui.side_input_2.isChecked(), 's3': self.ui.side_input_3.isChecked(), 's4': self.ui.side_input_4.text(), 's5': self.ui.side_input_5.text(), 's6': self.ui.side_input_6.text(), 's7': self.ui.side_input_7.text(), 's8': self.ui.side_input_8.isChecked(), 's9': self.ui.side_input_9.isChecked(), 's10': self.ui.side_input_10.text(), 's11': self.ui.side_input_11.text(), 's12': self.ui.side_input_12.text(), 's13': self.ui.side_input_13.text(), } if self.action_dialog == None: self.action_dialog = QtWidgets.QWidget() self.action_dialog.setWindowTitle("Action") self.action_dialog.ui = Ui_ActionDialog() self.action_dialog.ui.setupUi(self.action_dialog) self.action_dialog.setFixedHeight(223) self.action_dialog.setFixedWidth(515) self.action_dialog.ui.progressBar.setMinimum(0) self.action_dialog.ui.pushButton_start.clicked.connect(self.go_for_work) self.action_dialog.ui.pushButton_cancel.clicked.connect(self.action_dialog.close) self.action_dialog.ui.pushButton_finish.clicked.connect(self.action_dialog.close) self.action_dialog.ui.pushButton_start.setEnabled(True) self.action_dialog.ui.pushButton_cancel.setEnabled(True) self.action_dialog.ui.pushButton_finish.setEnabled(False) self.action_dialog.ui.label_desDir.setText(get_prefferd_text(i2)) action = 'Copying :' action_ = 'Copied' if i1 == 1: action = 'Moving :' action_ = 'Moved' self.action_dialog.ui.label_action_going.setText(action) self.action_dialog.ui.label_action.setText(action_) self.action_dialog.ui.stackedWidget.setCurrentIndex(1) self.action_dialog.ui.label_statBar.setText("""<html><body><p align="center"><span style="font-size:18pt;">Click 'Start' to go</span></p></body></html>""") self.action_dialog.ui.label_done.setText("0") self.action_dialog.ui.label_failed.setText("0") self.action_dialog.ui.label_notAllowed.setText("0") self.action_dialog.ui.label_ignored.setText("0") self.action_dialog.ui.progressBar.setValue(0) self.action_dialog.data_data = data self.action_dialog.data_folders = folders self.action_dialog.data_action = i1 self.action_dialog.data_des = i2 self.action_dialog.action_done = 0 self.action_dialog.destroy() self.action_dialog.show()
def select_output_folder(self): folder = str(QtWidgets.QFileDialog.getExistingDirectory(self, "Select Output Directory", directory=self.last_selected_out_dir)) if folder != '': folder = str(Path(folder)) self.last_selected_out_dir = join(*folder.split('\\')[:len(folder.split('\\'))-1]) self.ui.input_2.setText(folder)
def log(msg, stream=sys.stdout): stream.write(nowstr() + ' ' + msg + '\n') stream.flush() if len(sys.argv) != 4: print("usage: climateimport.py [newfile.dat] [siteid] [archivefile.dat]") # Get filename for import from cmdline def nowstr(): return datetime.now().strftime('%Y-%m-%d %H:%M') path = Path(sys.argv[1]) i = 0 log('Wait for ' + path.absolute) while not path.exists(): time.sleep(60) i += 1 if i > 20: log('After 20 tries does ' + path.basename + ' not exist', sys.stderr) sys.exit(1) try: # Get siteid from cmdline siteid = int(sys.argv[2]) log('Process settings for ' + path.basename) # Create a TextImport
def instrumentimport(self, filename, kwargs): """ Loads instrument data using a .conf file """ t0 = time.time() # Error streams errorstream = StringIO() # TODO: Major refactoring of this code logic, when to load gaps, etc. path = Path(web.abspath(filename.strip('/'))) print("path = %s" % path) import dataimport as di error = web.markdown(di.checkimport(path.absolute)) startdate = kwargs.get('startdate') enddate = kwargs.get('enddate') siteid = web.conv(int, kwargs.get('site')) instrumentid = web.conv(int, kwargs.get('instrument')) config = di.getconfig(path.absolute) if not config: errorstream.write( "No config available. Please provide a config for" " computing a decent result.") if config: valuetype = [e.valuetype for e in config.columns] if config: config.href = Path(config.filename).href if startdate: startdate = web.parsedate(startdate) if enddate: enddate = web.parsedate(enddate) stats = gaps = datasets = None sites = [] possible_datasets = [] if startdate and enddate: gaps = [(startdate, enddate)] if siteid and (instrumentid or config): absfile = web.abspath(filename.strip('/')) adapter = di.get_adapter(absfile, web.user(), siteid, instrumentid, startdate, enddate) adapter.errorstream = errorstream if 'loadstat' in kwargs: stats = adapter.get_statistic() startdate = min(v.start for v in stats.values()) enddate = max(v.end for v in stats.values()) if 'importdb' in kwargs and startdate and enddate: gaps = None datasets = di.importfile(absfile, web.user(), siteid, instrumentid, startdate, enddate) else: gaps = di.finddateGaps(siteid, instrumentid, valuetype, startdate, enddate) error = adapter.errorstream.getvalue() adapter.errorstream.close() t1 = time.time() log("Imported in %.2f s" % (t1 - t0)) return web.render('dbimport.html', di=di, error=error, filename=filename, instrumentid=instrumentid, dirlink=path.up(), siteid=siteid, gaps=gaps, stats=stats, datasets=datasets, config=config, #mmimport=isinstance(config, di.mm.ImportManualMeasurementsDescription), sites=sites, possible_datasets=possible_datasets)\ .render('html', doctype='html')