def get_article(request):
code = request.GET.get('code', None)
collection = request.GET.get('collection', None)
fmt = request.GET.get('format', 'json')
body = request.GET.get('body', 'false')
if not body in ['true', 'false']:
raise HTTPBadRequest("parameter 'metaonly' must be 'true' or 'false'")
body = asbool(body)
article = request.databroker.get_article(code,
collection=collection,
replace_journal_metadata=True,
body=body)
if article:
if fmt == 'xmlwos':
return Response(Export(article).pipeline_sci(),
content_type="application/xml")
if fmt == 'xmldoaj':
return Response(Export(article).pipeline_doaj(),
content_type="application/xml")
if fmt == 'xmlrsps':
return Response(Export(article).pipeline_rsps(),
content_type="application/xml")
if fmt == 'xmlpubmed':
return Response(Export(article).pipeline_pubmed(),
content_type="application/xml")
return Response(json.dumps(article), content_type="application/json")
def __init__(self, label, problem, options, src_dir=[], src_files=[]):
Export.__init__(self, label, problem, options)
source_dirs = ['point2point', 'point2point/admm', 'vehicles'] + src_dir
src_files = src_files + [
'ADMMPoint2Point.cpp', 'Point2Point.cpp', 'Vehicle.cpp'
]
if problem.options['separate_build']:
for updater in problem.updaters:
dest_dir = os.path.join(self.options['directory'],
str(updater)) + '/'
self.export(source_dirs, dest_dir, src_files,
updater.father_updx, updater, updater.problem)
else:
updaters = problem.separate_per_build()
if (len(updaters.keys()) == 1
and len(updaters.values()[0].keys()) == 1):
updater = updaters.values()[0].values()[0][0]
self.export(source_dirs, self.options['directory'], src_files,
updater.father_updx, updater, updater.problem)
else:
for veh_type, nghb_nr in updaters.items():
for nr, upd in nghb_nr.items():
dest_dir = os.path.join(self.options['directory'],
veh_type + '_' + nr + 'nghb')
self.export(source_dirs, dest_dir, src_files,
upd.father_updx, upd, upd.problem)
def export_to_csv(cls, export_data, file_name_path):
"""
let data to a csv file
"""
export = Export()
export.export_data_to_csv(export_data, file_name_path)
def __init__(self, app): self.app = app self._back = [] self._recursing = False self._next_callback = None self.setup_widgets() Export.__init__(self) ImportVdf.__init__(self) ImportSccprofile.__init__(self)
def __init__(self, problem, options):
Export.__init__(self, problem, options)
self.father = problem.father
self.problem = problem
if len(problem.vehicles) > 1:
raise ValueError(('Only export for single vehicle ' +
'problems is supported.'))
self.vehicle = problem.vehicles[0]
src_files = ['Point2Point.cpp', 'Vehicle.cpp']
self.export(['point2point', 'vehicles', 'tests/point2point'], self.options['directory'], src_files, problem.father, problem)
def test_export_tsv():
with app.app_context():
tsv_export = """Version 1\t\t\t\r
car Chia Van Jeff Van unassigned\r
driver Gideon Jeff\t\r
passenger Crystal Gabe\t\r
Version 2\t\t\t\r
car Chia Van Jeff Van unassigned\r
driver Gideon Jeff\r
passenger Crystal Gabe\r\n"""
print(Export.to_tsv(mock_arrangement()))
assert tsv_export == Export.to_tsv(mock_arrangement())
def test_export_csv():
with app.app_context():
csv_export = """Version 1,,,\r
car,Chia Van,Jeff Van,unassigned\r
driver,Gideon,Jeff,\r
passenger,Crystal,Gabe,\r
Version 2,,,\r
car,Chia Van,Jeff Van,unassigned\r
driver,Gideon,,Jeff\r
passenger,Crystal,,Gabe\r\n"""
print(Export.to_csv(mock_arrangement()))
assert csv_export == Export.to_csv(mock_arrangement())
def __init__(self, problem, options):
Export.__init__(self, problem, options)
self.father = problem.father
self.problem = problem
if len(problem.vehicles) > 1:
raise ValueError(
('Only export for single vehicle ' + 'problems is supported.'))
self.vehicle = problem.vehicles[0]
src_files = ['Point2Point.cpp', 'Vehicle.cpp']
self.export(['point2point', 'vehicles', 'tests/point2point'],
self.options['directory'], src_files, problem.father,
problem)
def test_export_to_html(self, mock_writer, mock_os, _):
mock_writer.write_temp_file.return_value = self.path
mock_os.path.basename.return_value = 'test'
mock_os.path.splitext.return_value = ['base1', 'base2']
mock_os.path.join.return_value = '/this/is/the/joined/path.html'
Export.export_html(self.document)
mock_writer.write_temp_file.assert_called_once_with(self.document)
mock_writer.write_exported_html_file.assert_called_once_with(
['markdown', self.path], '/this/is/the/joined/path.html')
mock_os.path.basename.assert_called_once_with(self.path)
mock_os.path.join.assert_called_once_with('/some/path', 'base1.html')
def init():
args = sys.argv[1:]
if len(args) == 0:
route = raw_input("> [route_parser] Insert a route to query with, or nothing to run a test route.] \n \
> ")
if len(args) == 0:
route = "4-40 Beach Dr Even#, 650-776 Mountjoy Ave Even#, 2019-2027 Runnymede Ave Odd# (19)"
print "[route_parser] No args provided - Running the following test route: "
print route
parsedroute = parse(route)
print "[route_parser] Parsed the test route: \n", parsedroute
ex = Export()
ex.set_login_details(None, None)
ex.set_parsed_routes(parsedroute)
ex.generate_excel()
print "[route_parser] Successfully parsed route description and wrote data to file."
return
wb = load_workbook(filename = '%s' % args[0], read_only=True)
ws = wb[wb.get_sheet_names()[0]]
print ws
result = []
for row in ws.iter_rows():
for cell in row:
if cell.column == 5 and cell.value != None:
print cell.value
print result
def _export_to_html(self, action):
"""
Menu activation handler for HTMl export.
Calls the `Export` module from `export.py` to handle the actual file manipulation.
"""
del action
doc = self.window.get_active_document()
if not doc:
print('[Export to HTML] No document.')
return
Export.export_html(doc)
def on_action_Export_to_odML_triggered(self):
"""
This method is called if you click on *File->Export to odML*.
Exports the virtual unit mappings to an odML file.
"""
filename = str(QtGui.QFileDialog.getSaveFileName(self, QtCore.QString("Choose a savename"), QtCore.QString(self._prodir)))
if filename:
try:
if filename.endswith(".odml"):
filename = filename[:-5]
Export.export_odml(filename, self._mystorage.load_map())
except IOError:
QtGui.QMessageBox.critical(self, "Export error", "The virtual unit maps could not be exported")
def get(self, project_id):
logger.debug("Training model " + str(project_id))
logger.debug("Validating project ID " + str(project_id))
result = self.ValidateData.validate_data(project_id)
if result != '':
logger.debug("Validation failed for the project ")
return {"status": "Error", "message": result}
# Set Project Status as "Training" so that all users can see model is under training
# Clear redis cache
r.delete("all_projects")
ProjectsModel.set_project_mode(mode="Training", project_id=project_id)
result = Export.call_main(project_id)
logger.debug(result)
# Start Training for the model
task_obj = trainer_app.send_task('tasks.train_model',
kwargs={'project_id': project_id})
logger.debug("Task ID " + str(task_obj.id))
# get status
status = trainer_app.AsyncResult(task_obj.id, app=trainer_app)
logger.debug("Status of the task " + str(status.state))
return {
"status": "Success",
"message": str(status.state),
"task_id": str(task_obj.id)
}
def test_export_to_pdf(self, mock_call, mock_writer, mock_os, _):
mock_writer.write_temp_file.return_value = self.path
mock_os.path.splitext.return_value = ['base1', 'base2']
call_args = 'lowriter --headless --convert-to pdf --outdir'.split(
' ') + ['/some/path', 'base1.html']
Export.export_pdf(self.document)
mock_writer.write_temp_file.assert_called_once_with(self.document)
mock_writer.write_exported_html_file.assert_called_once_with(
['markdown', self.path], 'base1.html')
mock_call.assert_called_once_with(call_args)
self.assertSequenceEqual(
mock_os.remove.call_args_list,
[mock.call(self.path),
mock.call('base1.html')])
def _create_spline_tf(self, father, problem):
defines = Export._create_spline_tf(self, father, problem)
for child, q_i in problem.q_i.items():
for name, ind in q_i.items():
if name in child._splines_prim:
basis = child._splines_prim[name]['basis']
for l in range(len(basis)):
sl_min = l * len(basis)
sl_max = (l + 1) * len(basis)
if set(range(sl_min, sl_max)) <= set(ind):
spl = child._splines_prim[name]
if spl['init'] is not None:
tf = '{'
for k in range(spl['init'].shape[0]):
tf += '{' + ','.join([
str(t)
for t in spl['init'][k].tolist()
]) + '},'
tf = tf[:-1] + '}'
defines.update({
('XVAR_%s_TF') % name.upper():
tf
})
break
return defines
def get_fragment_atoms_cartesian(fragment):
"""
returns the coordinates of the fragment as cartesian coords
as list of lists [['-2.7538', '15.9724', '22.6810'], ['0.7939', '16.3333', '21.3135'], ...
:param fragment:
:type fragment:
"""
from export import Export
ex = Export(gl, invert=False)
atoms = ex.format_atoms_for_export(gl.get_cell(fragment), gl.get_atoms(fragment), False)
coords = []
for i in atoms:
co = i.split()[2:5]
co = ['{:>.4f}'.format(float(x)) for x in co]
# print(co)
coords.append(co)
return coords
def export(self, file, fold_path=None):
exp = Export(file, self, fold_path)
f = open(file, "w")
f.write(exp.data)
f.close()
return
def get_arrangement(arrangement_id=None, export_type='json'):
if arrangement_id is None:
return ('', httplib.NO_CONTENT)
else:
arrangements = arrangementsMDB.get_arrangement_by_id(arrangement_id)
if len(arrangements) == 1:
return Export.export_arrangement(export_type, arrangements[0])
else:
return jsonify({"arrangement": "no arrangement found"})
def __init__(self, problem, options, src_dir=[], src_files=[]):
Export.__init__(self, problem, options)
source_dirs = ['point2point', 'point2point/admm', 'vehicles'] + src_dir
src_files = src_files + ['ADMMPoint2Point.cpp', 'Point2Point.cpp', 'Vehicle.cpp']
if problem.options['separate_build']:
for updater in problem.updaters:
dest_dir = os.path.join(self.options['directory'], str(updater)) + '/'
self.export(source_dirs, dest_dir, src_files, updater.father_updx, updater, updater.problem)
else:
updaters = problem.separate_per_build()
if (len(updaters.keys()) == 1 and len(updaters.values()[0].keys()) == 1):
updater = updaters.values()[0].values()[0][0]
self.export(source_dirs, self.options['directory'], src_files, updater.father_updx, updater, updater.problem)
else:
for veh_type, nghb_nr in updaters.items():
for nr, upd in nghb_nr.items():
dest_dir = os.path.join(self.options['directory'], veh_type+'_'+nr+'nghb')
self.export(source_dirs, dest_dir, src_files, upd.father_updx, upd, upd.problem)
def _create_initSplines(self, father, problem):
data = Export._create_initSplines(self, father, problem)
code = data['initSplines']
for child, q_i in problem.q_i.items():
for name, ind in q_i.items():
if name in child._splines_prim:
basis = child._splines_prim[name]['basis']
for l in range(len(basis)):
sl_min = l*len(basis)
sl_max = (l+1)*len(basis)
if set(range(sl_min, sl_max)) <= set(ind):
code += '\tsplines_tf["xvar_'+name+'"] = XVAR_'+name.upper()+'_TF;\n'
return {'initSplines': code}
def _create_initSplines(self, father, problem):
data = Export._create_initSplines(self, father, problem)
code = data['initSplines']
for child, q_i in problem.q_i.items():
for name, ind in q_i.items():
if name in child._splines_prim:
basis = child._splines_prim[name]['basis']
for l in range(len(basis)):
sl_min = l*len(basis)
sl_max = (l+1)*len(basis)
if set(range(sl_min, sl_max)) <= set(ind):
code += '\tsplines_tf["xvar_'+name+'"] = XVAR_'+name.upper()+'_TF;\n'
return {'initSplines': code}
def main():
info_db = YInfoDB(INFO_DB)
tpis_info = TPIs(TPI_DIR)
exports = Export(EXPORT)
pacs_info = PACs(PAC_DIR)
#for ttpi in tpis_info.get_all_tpi():
# print ttpi.data
c_price = ContractPrice(tpis_info, exports)
if not c_price.is_ok():
LOG.error("Error when generating contract price")
return
#pacs_info.show_pacs()
if c_price.process_pac(pacs_info) == False:
LOG.error("Error When processing PACs")
return
if c_price.process_infodb(info_db) == False:
LOG.error("Error when processing info db")
return
c_price.show_contract_price()
outbook = OutputWorkbook(TEMPLATE_FILE)
nanos, normal_pns, pc_pns = c_price.get_data()
ct = outbook.get_contract_sheet()
dt = outbook.get_detail_sheet()
it = outbook.get_invoice_sheet()
LOG.info("nano=%s", nanos)
LOG.info("normal=%s", normal_pns)
LOG.info("pc=%s", pc_pns)
#first process nano
process_nano(ct, dt, it, nanos, c_price, info_db)
#second normal
process_other(ct, dt, it, normal_pns, c_price)
#third, 微型电脑主机
process_other(ct, dt, it, pc_pns, c_price)
process_others(ct, dt, it, CONTRACT_ID, pacs_info, c_price)
outbook.save(DEST_FILE)
##now accouting
save_account(tpis_info, exports, c_price)
save_hawb_total(tpis_info, c_price)
print u"处理成功!!!!!!"
def create_constants(self, father, problem, point2point):
constants = {}
constants['std::string ADMMLBL'] = '"' + problem.label + '"'
constants['double RHO'] = problem.options['rho']
constants['int INITITER'] = problem.options['init_iter']
constants['int N_SHARED'] = problem.q_i_struct(0).cat.size(1)
constants['int N_NGHB'] = len(problem.fleet.get_neighbors(problem.vehicle))
constants['std::string UPDZPROBLEM'] = '"' + problem.problem_upd_z.name() + '"'
constants['std::string UPDLPROBLEM'] = '"' + problem.problem_upd_l.name() + '"'
constants['std::string UPDRESPROBLEM'] = '"' + problem.problem_upd_res.name() + '"'
data = Export.create_constants(self, father, problem, point2point)
code = data['constants']
for definition, value in constants.items():
code += 'const ' + str(definition) + ' = ' + str(value) + ';\n'
return {'constants': code}
def create_defines(self, father, problem, point2point):
defines = {}
defines['ADMMLBL'] = '"' + problem.label + '"'
defines['RHO'] = problem.options['rho']
defines['INITITER'] = problem.options['init_iter']
defines['N_SHARED'] = problem.q_i_struct(0).cat.size(1)
defines['N_NGHB'] = len(problem.fleet.get_neighbors(problem.vehicle))
defines['UPDZPROBLEM'] = '"' + problem.problem_upd_z.name() + '"'
defines['UPDLPROBLEM'] = '"' + problem.problem_upd_l.name() + '"'
defines['UPDRESPROBLEM'] = '"' + problem.problem_upd_res.name() + '"'
data = Export.create_defines(self, father, problem, point2point)
code = data['defines']
for name, define in defines.items():
code += '#define ' + str(name) + ' ' + str(define) + '\n'
return {'defines': code}
def __init__(self):
"""Creates the main UI dialog box and sets up the sections in the desired layout."""
QDialog.__init__(self)
self.font_type = "Courier"
self.font_size = 10
font = QFont(self.font_type, self.font_size)
font.setFixedPitch(True)
QApplication.setFont(font, "QPlainTextEdit")
self.ts_start = datetime.datetime.now()
self.ui_flags = FLAGS
self.dlg = uic.loadUi("dep.ui", self)
has_pickled_project = self.unpickle_project()
self.verbose = FLAGS.verbose
self.s3_ignore_fullsize_color = FLAGS.s3_ignore_fullsize_color
self.s3_sample_frame = FLAGS.s3_sample_frame
self.project_root = FLAGS.project_root
self.path_project = config.DOCKER_INPUT_ROOT
self.configure_farm()
self.full_size = self.dlg.frameSize()
verify_data.set_default_top_level_paths(self)
if not has_pickled_project:
verify_data.verify(self)
dep_util.set_full_size_widths(self)
self.calibrate = Calibration(self)
self.background = Background(self)
self.depth = DepthEstimation(self)
self.export = Export(self)
self.sections = [
self.calibrate, self.background, self.depth, self.export
]
global sections
sections = self.sections
self.setup_sections_layout()
self.setup_sections_signals()
self.setup_project()
self.setup_clock()
self.dlg.show()
def export_casadi_problems(self, destination, father, problem):
filenames = Export.export_casadi_problems(self, destination, father, problem)
obj = father.problem_description['obj']
con = father.problem_description['con']
var = father.problem_description['var']
par = father.problem_description['par']
opt = father.problem_description['opt']
nlp = {'x': var, 'p': par, 'f': obj, 'g': con}
options = {}
for key, value in opt['solver_options']['ipopt'].items():
options[key] = value
options.update({'expand': True})
solver = nlpsol('solver', 'ipopt', nlp, options)
solver.generate_dependencies('nlp.c')
cwd = os.getcwd()
shutil.move(cwd+'/nlp.c', destination+'src/nlp.c')
filenames.append('nlp.c')
return filenames
def _create_spline_tf(self, father, problem):
constants = Export._create_spline_tf(self, father, problem)
for child, q_i in problem.q_i.items():
for name, ind in q_i.items():
if name in child._splines_prim:
basis = child._splines_prim[name]['basis']
for l in range(len(basis)):
sl_min = l*len(basis)
sl_max = (l+1)*len(basis)
if set(range(sl_min, sl_max)) <= set(ind):
spl = child._splines_prim[name]
if spl['init'] is not None:
tf = '{'
for k in range(spl['init'].shape[0]):
tf += '{'+','.join([str(t) for t in spl['init'][k].tolist()])+'},'
tf = tf[:-1]+'}'
constants.update({('std::vector<std::vector<double>> XVAR_%s_TF') % name.upper(): tf})
break
return constants
def __init__(self, *args, **kwargs):
path, output, Format = args
file.exists(output)
files = []
self.findpythonfiles(path, files)
comments = []
self.findcomments(files, comments)
del files, path
self.clearpaths(comments)
Export(comments, output, Format)
def export_casadi_problems(self, destination, father, problem):
filenames = Export.export_casadi_problems(self, destination, father,
problem)
obj = father.problem_description['obj']
con = father.problem_description['con']
var = father.problem_description['var']
par = father.problem_description['par']
opt = father.problem_description['opt']
nlp = {'x': var, 'p': par, 'f': obj, 'g': con}
options = {}
for key, value in opt['solver_options']['ipopt'].items():
options[key] = value
options.update({'expand': True})
solver = nlpsol('solver', 'ipopt', nlp, options)
solver.generate_dependencies('nlp.c')
cwd = os.getcwd()
shutil.move(cwd + '/nlp.c', destination + 'src/nlp.c')
filenames.append('nlp.c')
return filenames
def export_result(self):
if not self.last_scan_type:
print("No result found, did you scan ?")
return
xporter = Export(self.opt['string']['exp_basename'])
if (self.opt['string']['export_format'] == "txt"):
xporter.export_txt(self.json_res, self.last_scan_type)
elif (self.opt['string']['export_format'] == "json"):
xporter.export_json(self.json_res)
else:
print("Unknow export format")
print("Available formats : txt, json")
def __init__(self, job_params):
self.task = job_params
try:
if self.task['crawl'] is True:
c = Crawler(self.task)
c.crawl()
elif self.task['report'] is True:
#crawtext.py report <project> [((--email=<email>| -e <email>) -u <user> -p <passwd>)| (-o <outfile> |--o=<outfile>)]
Report(self.task)
elif self.task['export'] is True:
#crawtext.py export [results|sources|logs|queue] <project> [(-o <outfile> |--o=<outfile>)] [-t <type> | --type=<type>]
Export(self.task)
# elif self.task['extract'] is True:
#new method for extract every url
except KeyError:
print self.task["project"]
print "Project %s not configured properly" % str(
self.task["project"])
s = Scheduler(self.task)
s.delete()
print "deleting project"
class DSR(object):
"""
main class
"""
def __init__(self, options):
"""
"""
import time
time1 = time.clock()
self.numpy_installed = False
# options from the commandline options parser:
self.options = options
self.external = False
self.fragment = ''
self.helpmsg = "*** Please ask [email protected] for help ***"
self.res_file = self.options.res_file
if self.options.external_restr:
self.external = True
self.res_file = self.options.external_restr
self.export_fragment = self.options.export_fragment
if self.export_fragment:
self.res_file = self.options.export_fragment
if self.export_fragment:
self.fragment = self.export_fragment
self.export_clip = self.options.export_clip
if self.export_clip:
self.fragment = self.export_clip
self.import_grade = self.options.import_grade
self.export_all = self.options.export_all
self.list_db = self.options.list_db
self.list_db_csv = self.options.list_db_csv
self.no_refine = self.options.no_refine
self.invert = self.options.invert
self.rigid = self.options.rigid_group
self.search_string = self.options.search_string
self.search_extern = self.options.search_extern
self.head_csv = self.options.head_for_gui
if self.head_csv:
self.fragment = self.head_csv
if self.options.selfupdate:
import selfupdate
selfupdate.update_dsr()
sys.exit()
#################################
self.maindb_path = ''
self.userdb_path = ''
try:
self.set_database_locations()
self.gdb = ParseDB(self.maindb_path, self.userdb_path)
except Exception as e: # @UnusedVariable
print("*** Initializing the database failed ***")
raise
# List of database Fragments:
if self.list_db_csv:
print('DSR version: {}'.format(VERSION))
for i in self.gdb.list_fragments():
print('{};;{};;{};;{}'.format(i[0], i[3], i[1], i[2]))
sys.exit()
try:
from export import Export
self.export = Export(gdb=self.gdb, invert=self.invert)
except Exception as e:
print("*** Unable to export informations from DSR ***")
print(e)
sys.exit()
#################################
if self.head_csv:
self.head_to_gui()
if self.search_extern:
result = search_fragment_name(self.search_extern, self.gdb, numresults=7)
for i in result:
print('{};;{};;{};;{}'.format(i[0], i[1], i[2], i[3]))
sys.exit()
print(program_name)
if self.list_db:
self.list_dbentries()
if self.search_string:
result = search_fragment_name(self.search_string, self.gdb, numresults=7)
print_search_results(result)
sys.exit()
# Export !all! fragments
if self.export_all:
self.export.export_all_fragments()
# Export one fragment
if self.export_fragment:
print('Exporting "{0}" to {0}.res'.format(self.fragment))
self.fragment = self.export_fragment
try:
self.export.write_res_file(self.fragment)
except:
raise
sys.exit()
if self.export_clip:
try:
self.export.export_to_clip(self.fragment)
except:
raise
sys.exit()
# Import a GRADE fragment
if self.import_grade:
mog = ImportGRADE(self.import_grade, self.gdb, self.invert, self.gdb.maindb_path, self.gdb.userdb_path)
mog.write_user_database()
sys.exit()
if not any(list(vars(self.options.all_options).values()) + [self.res_file]):
self.options.error()
if not self.res_file:
self.options.error()
self.rl = ResList(self.res_file)
self.reslist = self.rl.get_res_list()
try:
import numpy as np
self.numpy_installed = True
except ImportError:
print('*** Numpy was not found. Please reinstall DSR or install the numpy package in order to use DSR. ***')
sys.exit()
self.main()
time2 = time.clock()
runtime = (time2 - time1)
print('Runtime: {:>.1f} s'.format(runtime))
print('DSR run complete.')
###############################################################################
def set_database_locations(self):
"""
Tries to find the database files in their default locations after a regular DSR installation.
Returns
-------
bool
"""
if not self.userdb_path:
# using the environment variable turned out to be too complicated.
homedir = os.path.expanduser("~")
self.userdb_path = os.path.join(homedir, "dsr_user_db.txt")
if not os.path.isfile(self.userdb_path):
touch(self.userdb_path)
if not self.maindb_path:
try:
main_dbdir = os.environ["DSR_DIR"]
self.maindb_path = os.path.join(main_dbdir, 'dsr_db.txt')
except KeyError:
main_dbdir = './'
self.maindb_path = os.path.join(main_dbdir, 'dsr_db.txt')
return True
def head_to_gui(self):
"""
Exports current fragment header and atoms to the GUI
"""
atoms = []
try:
atoms = self.export.export_to_gui(self.fragment)
except Exception as e:
# print(e)
print("*** Could not get atom information ***")
print(self.helpmsg)
sys.exit()
print("\n<atoms>")
print(atoms)
print("</atoms>")
# prints most of the needed info:
self.gdb.get_head_for_gui(self.fragment)
sys.exit()
def list_dbentries(self):
"""
list all entries in the db.
"""
dbdir = os.path.expanduser('~')
fragnames = []
num = 0
try:
(width, _) = get_terminal_size()
except():
width = 80
print('\n Entries found in the databases:\n')
print(' Fragment | Line | DB Name | Full name, Comments ')
print(sep_line)
for num, line in enumerate(self.gdb.list_fragments()):
fragnames.append(line[0])
line = ' {:<17}| {:<5}| {:<11}| {}'.format(*line)
print(line[:width - 1])
print('\n {} Fragments in the database(s).'.format(num),
'\n Feel free to add more fragments to "{}dsr_user_db.txt"'.format(dbdir + os.path.sep))
for fragment in fragnames:
self.gdb.check_consistency(fragment)
self.gdb.check_db_atom_consistency(fragment)
self.gdb.check_db_restraints_consistency(fragment)
self.gdb.check_sadi_consistence(fragment)
from selfupdate import is_update_needed
if is_update_needed(silent=True):
print("\n*** An update for DSR is available. You can update with 'dsr -u' ***")
sys.exit()
def main(self):
"""
main object to run DSR as command line program
"""
dbatoms = []
# The database content:
import atomhandling
basefilename = filename_wo_ending(self.res_file)
if not basefilename:
print('*** Illegal option ***')
sys.exit()
if len(self.reslist) == 0:
print("*** The input file is empty. Can not proceed! ***")
sys.exit()
find_atoms = atomhandling.FindAtoms(self.reslist)
rle = ResListEdit(self.reslist, find_atoms)
dsrp = DSRParser(self.reslist)
self.fragment = dsrp.fragment
restraints = self.gdb.get_restraints(self.fragment) # this is only executed once
db_residue_string = self.gdb.get_resi(self.fragment)
dbatoms = self.gdb.get_atoms(self.fragment, self.invert) # only the atoms of the dbentry as list
# the atomtypes of the dbentry as list e.g. ['C', 'N', ...]
db_atom_types = atomhandling.get_atomtypes(dbatoms)
sf = atomhandling.SfacTable(self.reslist, db_atom_types)
sfac_table = sf.set_sfac_table() # from now on this sfac table is set
resi = Resi(dsrp, db_residue_string, find_atoms)
# line where the dsr command is found in the resfile:
if dsrp.cf3_active:
from cf3fit import CF3
cf3 = CF3(rle, find_atoms, self.reslist, self.fragment, sfac_table,
basefilename, dsrp, resi, self.res_file, self.options)
if self.fragment == 'cf3':
cf3.cf3(afix='130')
if self.fragment == 'cf6':
cf3.cf3(afix='120')
if self.fragment == 'cf9':
cf3.cf9()
print('\nFinished...')
sys.exit()
# checks have to be after CF3, CF6 etc.
self.gdb.check_consistency(self.fragment)
self.gdb.check_db_atom_consistency(self.fragment)
self.gdb.check_db_restraints_consistency(self.fragment)
self.gdb.check_sadi_consistence(self.fragment)
if dsrp.occupancy:
rle.set_free_variables(dsrp.occupancy)
restraints = remove_resi(restraints)
# corrects the atom type according to the previous defined global sfac table:
dbatoms = atomhandling.set_final_db_sfac_types(db_atom_types, dbatoms, sfac_table)
if not dsrp.unit_line:
print('*** No UNIT instruction in res file found! Can not proceed! ***')
print('Inserting {} into res File.'.format(self.fragment))
if self.invert:
print('Fragment inverted.')
print('Source atoms: {}'.format(', '.join(dsrp.source)))
print('Target atoms: {}'.format(', '.join(dsrp.target)))
shx = ShelxlRefine(self.reslist, basefilename, find_atoms, self.options)
shx.backup_shx_file()
# several checks if the atoms in the dsr command line are consistent
atomhandling.check_source_target(dsrp.source, dsrp.target, dbatoms)
num = atomhandling.NumberScheme(self.reslist, dbatoms, dsrp)
# returns also the atom names if residue is active
fragment_numberscheme = num.get_fragment_number_scheme()
print('Fragment atom names: {}'.format(', '.join(fragment_numberscheme)))
dfix_head = ''
if dsrp.dfix:
restr = Restraints(self.fragment, self.gdb)
dfix_12 = restr.get_formated_12_dfixes()
dfix_13 = restr.get_formated_13_dfixes()
flats = restr.get_formated_flats()
restraints = dfix_12 + dfix_13 + flats
# ##########Not using SHELXL for fragment fit: ###########
print("--- Using fast fragment fit ---")
if self.options.target_coords:
target_coords = chunks(self.options.target_coords, 3)
else:
# {'C1': ['1.123', '0.7456', '3.245']}
target_coordinates = find_atoms.get_atomcoordinates(dsrp.target)
target_coords = [target_coordinates[key] for key in dsrp.target]
# Uppercase is important here to avoid KeyErrors in source_atoms generation
atnames = self.gdb.get_atomnames(self.fragment, uppercase=True)
source_atoms = dict(zip(atnames, self.gdb.get_coordinates(self.fragment, cartesian=True, invert=self.invert)))
# Coordinates only from the source, not the entire fragment:
source_coords = [source_atoms[x] for x in dsrp.source]
target_coords = [frac_to_cart(x, rle.get_cell()) for x in target_coords]
from rmsd import fit_fragment
# The source and target atom coordinates are fitted first. Then The complete fragment
# is rotated and translated to the target position as calculated before.
# parameter cartiesian has to be false here:
fragment_coords = self.gdb.get_coordinates(self.fragment, cartesian=False, invert=self.invert)
fitted_fragment, rmsd = fit_fragment(fragment_coords,
source_atoms=source_coords,
target_atoms=target_coords)
# Moving back to the position of the first atom to have a reference:
import numpy as np
from rmsd import centroid
# I have to make sure that I use the centroid of the correct atoms from target and source,
# otherwise the fragment is shifted to a wrong position.
# The third atom from the fragment e.g. has to be the third from the fragment to get
# the correct centroid:
center_difference = centroid(np.array(target_coords)) - \
centroid(np.array([list(fitted_fragment)[atnames.index(dsrp.source[x])]
for x in range(len(source_coords))]))
# finishing shift to correct centroid:
fitted_fragment += center_difference
# Or even lower than 0.1?
if rmsd < 0.1:
print('Fragment fit successful with RMSD of: {:8.3}'.format(rmsd))
else:
print('*** Fragment fit might have failed with RMSD of: {:8.3} ***'.format(rmsd))
fitted_fragment = [cart_to_frac(x, rle.get_cell()) for x in fitted_fragment]
afix_entry = []
e2s = Elem_2_Sfac(sfac_table)
for at, coord, atype in zip(fragment_numberscheme, fitted_fragment, db_atom_types):
sfac_num = str(e2s.elem_2_sfac(atype))
if dsrp.occupancy:
occ = float(dsrp.occupancy)
else:
occ = 11.0
afix_entry.append(isoatomstr.format(at, sfac_num, coord[0], coord[1], coord[2], occ, 0.03))
afix_entry = "\n".join(afix_entry)
new_atomnames = list(reversed(fragment_numberscheme))
same_resi = ''
if not dsrp.resiflag:
restraints = rename_restraints_atoms(new_atomnames, self.gdb.get_atomnames(self.fragment), restraints)
else:
restraints = resi.format_restraints(restraints)
# SADI\n
same_resi = ["SAME_{} {} > {}\n".format(resi.get_residue_class, new_atomnames[-1], new_atomnames[0])]
# Adds a "SAME_resiclass firstatom > lastatom" to the afix:
if not self.options.rigid_group:
restraints += same_resi
# if dsrp.resiflag: # <- Or should I do this?
restraints += ["SIMU 0.04 0.08 1"]
if not options.external_restr:
restraints = remove_duplicate_restraints(self.reslist, restraints, resi.get_residue_class)
restraints = wrap_headlines(restraints)
dfx_file_name = ''
if dsrp.part:
afix_entry = "PART {} {}\n".format(dsrp.part, dsrp.occupancy) + afix_entry + "\nPART 0"
if dsrp.resiflag:
afix_entry = 'RESI {} {}\n{}\nRESI 0'.format(resi.get_residue_class, resi.get_resinumber, afix_entry)
if self.options.rigid_group:
afix_entry = 'AFIX 9\n' + afix_entry
if options.external_restr and not self.rigid:
pname, ext = os.path.splitext(basefilename + '.dfix')
if dsrp.dfix:
dfx_file_name = pname + "_dfx" + ext
else:
dfx_file_name = pname + ext
dfx_file_name = write_dbhead_to_file(dsrp, dfx_file_name, restraints, resi.get_residue_class,
resi.get_resinumber)
if dsrp.resiflag:
restraints = 'REM Restraints for residue {}:\n+{}\n' \
.format(resi.get_residue_class, dfx_file_name)
else:
restraints = 'REM Restraints for DSR fragment:\n+{}\n' \
.format(dfx_file_name)
if self.options.rigid_group:
afix_entry += '\nAFIX 0\n'
# Adds the origin of restraints and fragment to res file:
import textwrap
source = textwrap.wrap("REM Restraints for Fragment {}, {} from: {}. "
"Please cite https://doi.org/10.1107/S1600576718004508".format(
self.fragment,
self.gdb.get_fragment_name(self.fragment),
self.gdb.get_src(self.fragment)),
width=74, subsequent_indent='REM ')
source = '\n'.join(source) + '\n'
# check if restraints already inserted:
for line in self.reslist:
try:
if line.split()[4] == self.fragment + ',':
source = ''
break
except IndexError:
continue
# + 'AFIX 0\n' before hklf seems to be not needed after shelx-2013:
self.reslist[dsrp.hklf_line - 1] = self.reslist[dsrp.hklf_line - 1] + afix_entry + '\n'
if not self.rigid:
self.reslist[dsrp.unit_line] = self.reslist[dsrp.unit_line] + source + ''.join(restraints)
# write to file:
self.rl.write_resfile(self.reslist, '.res')
if dsrp.command == 'REPLACE':
print("Replace mode active\n")
self.rl = ResList(self.res_file)
reslist = self.rl.get_res_list()
self.reslist, find_atoms = atomhandling.replace_after_fit(self.rl, reslist, resi,
fragment_numberscheme, rle.get_cell())
self.rl.write_resfile(self.reslist, '.res')
os.remove(shx.backup_file)
def set_default_options(self):
Export.set_default_options(self)
self.options['executable'] = 'Point2Point'
def create_functions(self, father, problem, point2point):
code = Export.create_functions(self, father, problem, point2point)
code.update(self._create_retrieveSharedVariables(father, problem))
code.update(self._create_transformSharedSplines(father, problem, point2point))
return code
def __init__(self, app):
QMainWindow.__init__(self) #QMainWindow.__init__(self, None, Qt.FramelessWindowHint)
self.setupUi(self)
self.app = app
self.settings = Settings(self)
self.resizeElements()
self.darkstyle = self.genDarkStyle()
self.def_style = """
QWidget { font-size: 10pt; font-family: Consolas}
QLineEdit { font-size: 13pt; font-family: Consolas}
"""
if self.settings["theme"] == "dark":
self.dark_theme = True
self.style = self.darkstyle
else:
self.dark_theme = False
self.style = self.def_style
self.app.setStyleSheet(self.style)
self.appversion = appversion
self.setupTable()
self.export = Export(self)
self.actionPublic_Mode.setChecked(self.settings["public_mode"])
self.factor.setValue(self.settings["zoom_factor"])
self.ocr_all_set = False
self.color_image = None
self.preview_image = None
self.current_result = None
self.newupd = None
self.zoom = False
self.minres = 0
self.busyDialog = None
self.fields = [self.name, self.sell, self.buy, self.demand_num, self.demand,
self.supply_num, self.supply]
self.canvases = [self.name_img, self.sell_img, self.buy_img, self.demand_img,
self.demand_text_img, self.supply_img, self.supply_text_img]
#setup buttons
self.add_button.clicked.connect(self.addFiles)
self.remove_button.clicked.connect(self.removeFile)
self.remove_all_button.clicked.connect(self.removeAllFiles)
self.add_all_button.clicked.connect(self.addAllScreenshots)
self.save_button.clicked.connect(self.addItemToTable)
self.skip_button.clicked.connect(self.nextLine)
self.continue_button.clicked.connect(self.continueOCR)
self.ocr_button.clicked.connect(self.performOCR)
self.ocr_all.clicked.connect(self.runOCRAll)
self.export_button.clicked.connect(self.export.exportToFile)
self.bpc_button.clicked.connect(self.export.bpcExport)
self.eddn_button.clicked.connect(self.export.eddnExport)
self.clear_table.clicked.connect(self.clearTable)
self.zoom_button.clicked.connect(self.drawOCRPreview)
QObject.connect(self.actionHelp, SIGNAL('triggered()'), self.openHelp)
QObject.connect(self.actionUpdate, SIGNAL('triggered()'), self.openUpdate)
QObject.connect(self.actionAbout, SIGNAL('triggered()'), self.About)
QObject.connect(self.actionOpen, SIGNAL('triggered()'), self.addFiles)
QObject.connect(self.actionPreferences, SIGNAL('triggered()'), self.openSettings)
QObject.connect(self.actionPublic_Mode, SIGNAL('triggered()'), self.toggleMode)
QObject.connect(self.actionCommodity_Editor, SIGNAL('triggered()'), self.openEditor)
self.error_close = False
#set up required items for nn
self.training_image_dir = unicode(self.settings.app_path.decode('windows-1252'))+os.sep+u"nn_training_images"+os.sep
self.loadPlugins()
self.restorePos()
self.eddnthread = EDDNExport(self)
QObject.connect(self.eddnthread, SIGNAL('finished(QString)'), self.export.eddnFinished)
QObject.connect(self.eddnthread, SIGNAL('update(int,int)'), self.export.eddnUpdate)
self.thread = Worker()
self.connect(self.thread, SIGNAL("output(QString, QString)"), self.showUpdateAvailable)
self.thread.check(self.appversion)
if not self.settings.reg.contains('info_accepted'):
self.infoDialog = InfoDialog()
self.infoDialog.exec_()
else:
if not self.settings['info_accepted']:
self.infoDialog = InfoDialog()
self.infoDialog.exec_()
self.checkAppConfigXML()
def __init__(self, options):
"""
"""
import time
time1 = time.clock()
self.numpy_installed = False
# options from the commandline options parser:
self.options = options
self.external = False
self.fragment = ''
self.helpmsg = "*** Please ask [email protected] for help ***"
self.res_file = self.options.res_file
if self.options.external_restr:
self.external = True
self.res_file = self.options.external_restr
self.export_fragment = self.options.export_fragment
if self.export_fragment:
self.res_file = self.options.export_fragment
if self.export_fragment:
self.fragment = self.export_fragment
self.export_clip = self.options.export_clip
if self.export_clip:
self.fragment = self.export_clip
self.import_grade = self.options.import_grade
self.export_all = self.options.export_all
self.list_db = self.options.list_db
self.list_db_csv = self.options.list_db_csv
self.no_refine = self.options.no_refine
self.invert = self.options.invert
self.rigid = self.options.rigid_group
self.search_string = self.options.search_string
self.search_extern = self.options.search_extern
self.head_csv = self.options.head_for_gui
if self.head_csv:
self.fragment = self.head_csv
if self.options.selfupdate:
import selfupdate
selfupdate.update_dsr()
sys.exit()
#################################
self.maindb_path = ''
self.userdb_path = ''
try:
self.set_database_locations()
self.gdb = ParseDB(self.maindb_path, self.userdb_path)
except Exception as e: # @UnusedVariable
print("*** Initializing the database failed ***")
raise
# List of database Fragments:
if self.list_db_csv:
print('DSR version: {}'.format(VERSION))
for i in self.gdb.list_fragments():
print('{};;{};;{};;{}'.format(i[0], i[3], i[1], i[2]))
sys.exit()
try:
from export import Export
self.export = Export(gdb=self.gdb, invert=self.invert)
except Exception as e:
print("*** Unable to export informations from DSR ***")
print(e)
sys.exit()
#################################
if self.head_csv:
self.head_to_gui()
if self.search_extern:
result = search_fragment_name(self.search_extern, self.gdb, numresults=7)
for i in result:
print('{};;{};;{};;{}'.format(i[0], i[1], i[2], i[3]))
sys.exit()
print(program_name)
if self.list_db:
self.list_dbentries()
if self.search_string:
result = search_fragment_name(self.search_string, self.gdb, numresults=7)
print_search_results(result)
sys.exit()
# Export !all! fragments
if self.export_all:
self.export.export_all_fragments()
# Export one fragment
if self.export_fragment:
print('Exporting "{0}" to {0}.res'.format(self.fragment))
self.fragment = self.export_fragment
try:
self.export.write_res_file(self.fragment)
except:
raise
sys.exit()
if self.export_clip:
try:
self.export.export_to_clip(self.fragment)
except:
raise
sys.exit()
# Import a GRADE fragment
if self.import_grade:
mog = ImportGRADE(self.import_grade, self.gdb, self.invert, self.gdb.maindb_path, self.gdb.userdb_path)
mog.write_user_database()
sys.exit()
if not any(list(vars(self.options.all_options).values()) + [self.res_file]):
self.options.error()
if not self.res_file:
self.options.error()
self.rl = ResList(self.res_file)
self.reslist = self.rl.get_res_list()
try:
import numpy as np
self.numpy_installed = True
except ImportError:
print('*** Numpy was not found. Please reinstall DSR or install the numpy package in order to use DSR. ***')
sys.exit()
self.main()
time2 = time.clock()
runtime = (time2 - time1)
print('Runtime: {:>.1f} s'.format(runtime))
print('DSR run complete.')
def export(self, _):
Export(self)
def test_basic_export():
ex = Export()
ex.set_login_details(None, None)
searchdict = {"City" : "Colwood"}
ex.set_search_dict(searchdict)
ex.generate_excel()
def __init__(self):
self.config = Config()
self.Loader = Loader()
self.Export = Export()
self.DataProcess = DataProcess()
def test_basic_export():
ex = Export()
ex.set_login_details(None, None)
searchdict = {"City": "Colwood"}
ex.set_search_dict(searchdict)
ex.generate_excel()
CustomActionsModel = CustomActionsModel()
ProjectsModel = ProjectsModel()
CopyProjectModel = CopyProjectModel()
DomainsModel = DomainsModel()
ConversationsModel = ConversationsModel()
RefreshDbModel = RefreshDbModel()
IntentsModel = IntentsModel()
IntentDetailModel = IntentDetailModel()
ResponseModel = ResponseModel()
ResponseDetailModel = ResponseDetailModel()
StoryDetailModel = StoryDetailModel()
StoryModel = StoryModel()
EntityModel = EntityModel()
ExportProjectModel = ExportProjectModel()
ImportProjectModel = ImportProjectModel()
Export = Export()
# Setting Expiry for redis cache
GLOBAL_EXPIRY = 60
# Initiate redis
try:
r = redis.Redis(host=os.environ['REDIS_URL'],
port=os.environ['REDIS_PORT'],
charset="utf-8",
decode_responses=True)
logger.info("Trying to connect to Redis Docker container ")
except KeyError:
logger.debug("Local run connecting to Redis ")
r = redis.Redis(host='localhost',
class Course_Initialization(object):
'''
課程初始化
'''
def __init__(self):
self.config = Config()
self.Loader = Loader()
self.Export = Export()
self.DataProcess = DataProcess()
def set_course_data_path(self, dic_path):
print("start to initialization of path.")
self.config.set_config_path('student_grade_csv_path', dic_path, "student_grade.csv" )
self.config.set_config_path('survey_csv_path', dic_path, "survey.csv")
self.config.set_config_path('zip_path', dic_path, "log.zip")
self.config.set_config_path('log_dic_path', dic_path, "log")
self.config.set_config_path('log_path', dic_path, "log.csv")
self.config.set_config_path('log_csv_path', dic_path, "log_data.csv")
self.config.set_config_path('clean_data_csv_path', dic_path, "clean_data.csv")
self.config.set_config_path('analysis_data_csv_path', dic_path, "analysis_data.csv")
self.config.set_config_path('student_pass_csv_path', dic_path, "student_pass.csv")
def initialization(self, dic_path):
'''
初始化課程資訊
'''
print("*******************************開始初始化*********************************")
# 整理出學生是否通過成績
try:
student_grade_csv_path = self.config.get_student_grade_csv_path()
grade_df = self.Loader.load_csv(student_grade_csv_path)
except:
print(dic_path + " has no " + self.config.get_student_grade_csv_path())
grade_data = grade_df[['Student ID','Grade']]
grade_data.rename(columns={'Student ID':'user_id'}, inplace=True)
grade_data['target'] = grade_data.apply(self.DataProcess.get_pass, axis=1)
grade_data = grade_data.drop('Grade', axis=1)
grade_data = grade_data.sort_values("user_id").reset_index(drop=True)
self.Export.export_data_to_csv(grade_data, dic_path+'/student_pass.csv')
print("student_pass.csv processing completed.")
print("****************************************************************")
# 整理出 Log data
try:
self.Loader.unzip(self.config.get_zip_path())
log_dic_path = self.config.get_log_dic_path()
self.Export.export_log_data_from_zip(log_dic_path, dic_path)
except:
print(dic_path + " has no " + self.config.get_log_dic_path())
print("log_data.csv processing completed.")
print("****************************************************************")
# 整理出 clean data
try:
log_csv_path = self.config.get_log_csv_path()
print(log_csv_path)
log_data = self.Loader.load_csv(log_csv_path)
except:
print(dic_path + " has no " + self.config.get_log_csv_path())
self.Export.export_clean_data_to_csv(log_data, dic_path)
print("clean_data.csv processing completed.")
print("****************************************************************")
# 整理出 analysis_data
try:
clean_data_csv_path = self.config.get_clean_data_csv_path()
clean_data = self.Loader.load_csv(clean_data_csv_path)
except:
print(dic_path + " has no " + self.config.get_clean_data_csv_path())
self.Export.export_analysis_data_to_csv(clean_data, dic_path)
print("analysis_data.csv processing completed.")
def test_parse_export():
ex = Export()
ex.set_login_details(None, None)
parsedroutes = route_parser.parse("4-40 Beach Dr Even#, 650-776 Mountjoy Ave Even#, 2019-2027 Runnymede Ave Odd# (19)")
ex.set_parsed_routes(parsedroutes)
ex.generate_excel()
def export_file_action(self):
dialog = Export(parent=self, df=self.table)
if dialog.exec_():
dialog.export_to_file()
def test_basic_export():
ex = Export()
ex.set_login_details(None, None)
ex.set_parsed_routes([["SELLECK WAY",[3223]]])
ex.generate_excel()
def execute(self, parameters, messages):
"""Runs the script"""
# Get the user's input
fc = parameters[0].valueAsText
field_mappings = parameters[1].valueAsText
fields = parameters[1].valueAsText.split(';')
fields.append('SHAPE@XY')
output_dir = parameters[2].valueAsText
output_name = parameters[3].valueAsText
convert_to_wgs84 = self.toBool(parameters[4].valueAsText)
convert_to_geojson = self.toBool(parameters[5].valueAsText)
convert_to_kmz = self.toBool(parameters[6].valueAsText)
convert_to_csv = self.toBool(parameters[7].valueAsText)
convert_metadata = self.toBool(parameters[8].valueAsText)
debug = self.toBool(parameters[9].valueAsText)
# Setup vars
output_path = output_dir + '\\' + output_name
shp_output_path = output_dir + '\\shapefile'
shp_temp_output_path = output_dir + '\\shapefile\\temp\\'
shapefile = shp_output_path + '\\' + output_name + '.shp'
temp_shapefile = shp_output_path + '\\temp\\' + output_name + '.shp'
if debug:
AddMessage('Field infos:')
AddMessage(field_mappings)
try:
arcpy.Delete_management('temp_layer')
except:
if debug:
AddMessage('Did not have a temp_layer feature ' +
'class to delete')
if not os.path.exists(shp_output_path):
os.makedirs(shp_output_path)
if debug:
AddMessage('Created directory ' + shp_output_path)
if not os.path.exists(shp_temp_output_path):
os.makedirs(shp_temp_output_path)
else:
for file in os.listdir(shp_temp_output_path):
file_path = os.path.join(shp_temp_output_path, file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except:
AddWarning('Unable to delete ' + file +
'from the temp folder. This ' +
'may become a problem later')
pass
arcpy.MakeFeatureLayer_management(fc, 'temp_layer', '', '',
field_mappings)
arcpy.CopyFeatures_management('temp_layer', temp_shapefile)
if convert_to_wgs84:
AddMessage('Converting spatial reference to WGS84...')
arcpy.Project_management(temp_shapefile, shapefile, "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433],METADATA['World',-180.0,-90.0,180.0,90.0,0.0,0.0174532925199433,0.0,1262]]", "WGS_1984_(ITRF00)_To_NAD_1983", "PROJCS['NAD_1983_StatePlane_Pennsylvania_South_FIPS_3702_Feet',GEOGCS['GCS_North_American_1983',DATUM['D_North_American_1983',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Lambert_Conformal_Conic'],PARAMETER['False_Easting',1968500.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',-77.75],PARAMETER['Standard_Parallel_1',39.93333333333333],PARAMETER['Standard_Parallel_2',40.96666666666667],PARAMETER['Latitude_Of_Origin',39.33333333333334],UNIT['Foot_US',0.3048006096012192]]")
AddMessage('Projection conversion completed.')
else:
AddMessage('Exporting shapefile already in WGS84...')
arcpy.FeatureClassToShapefile_conversion(temp_shapefile,
shp_output_path)
try:
arcpy.Delete_management('temp_layer')
except:
AddError('Unable to delete in_memory feature class')
AddMessage('Compressing the shapefile to a .zip file...')
export = Export(output_dir, output_name, debug)
zip = export.zip()
if zip:
AddMessage('Finished creating ZIP archive')
if convert_to_geojson:
AddMessage('Converting to GeoJSON...')
output = output_path + '.geojson'
geojson = esri2open.toOpen(shapefile, output,
includeGeometry='geojson')
if geojson:
AddMessage('Finished converting to GeoJSON')
if convert_to_kmz:
AddMessage('Converting to KML...')
kmz = export.kmz()
if kmz:
AddMessage('Finished converting to KMZ')
if convert_to_csv:
AddMessage('Converting to CSV...')
csv = export.csv()
if csv:
AddMessage('Finished converting to CSV')
if convert_metadata:
AddMessage('Converting metadata to Markdown ' +
'README.md file...')
md = export.md()
if md:
AddMessage('Finished converting metadata to ' +
'Markdown README.md file')
# Delete the /temp directory because we're done with it
shutil.rmtree(shp_output_path + '\\temp')
if (debug):
AddMessage('Deleted the /temp folder because we don\'t' +
' need it anymore')
return
class EliteOCR(QMainWindow, Ui_MainWindow):
def __init__(self, app):
QMainWindow.__init__(self) #QMainWindow.__init__(self, None, Qt.FramelessWindowHint)
self.setupUi(self)
self.app = app
self.settings = Settings(self)
self.resizeElements()
self.darkstyle = self.genDarkStyle()
self.def_style = """
QWidget { font-size: 10pt; font-family: Consolas}
QLineEdit { font-size: 13pt; font-family: Consolas}
"""
if self.settings["theme"] == "dark":
self.dark_theme = True
self.style = self.darkstyle
else:
self.dark_theme = False
self.style = self.def_style
self.app.setStyleSheet(self.style)
self.appversion = appversion
self.setupTable()
self.export = Export(self)
self.actionPublic_Mode.setChecked(self.settings["public_mode"])
self.factor.setValue(self.settings["zoom_factor"])
self.ocr_all_set = False
self.color_image = None
self.preview_image = None
self.current_result = None
self.newupd = None
self.zoom = False
self.minres = 0
self.busyDialog = None
self.fields = [self.name, self.sell, self.buy, self.demand_num, self.demand,
self.supply_num, self.supply]
self.canvases = [self.name_img, self.sell_img, self.buy_img, self.demand_img,
self.demand_text_img, self.supply_img, self.supply_text_img]
#setup buttons
self.add_button.clicked.connect(self.addFiles)
self.remove_button.clicked.connect(self.removeFile)
self.remove_all_button.clicked.connect(self.removeAllFiles)
self.add_all_button.clicked.connect(self.addAllScreenshots)
self.save_button.clicked.connect(self.addItemToTable)
self.skip_button.clicked.connect(self.nextLine)
self.continue_button.clicked.connect(self.continueOCR)
self.ocr_button.clicked.connect(self.performOCR)
self.ocr_all.clicked.connect(self.runOCRAll)
self.export_button.clicked.connect(self.export.exportToFile)
self.bpc_button.clicked.connect(self.export.bpcExport)
self.eddn_button.clicked.connect(self.export.eddnExport)
self.clear_table.clicked.connect(self.clearTable)
self.zoom_button.clicked.connect(self.drawOCRPreview)
QObject.connect(self.actionHelp, SIGNAL('triggered()'), self.openHelp)
QObject.connect(self.actionUpdate, SIGNAL('triggered()'), self.openUpdate)
QObject.connect(self.actionAbout, SIGNAL('triggered()'), self.About)
QObject.connect(self.actionOpen, SIGNAL('triggered()'), self.addFiles)
QObject.connect(self.actionPreferences, SIGNAL('triggered()'), self.openSettings)
QObject.connect(self.actionPublic_Mode, SIGNAL('triggered()'), self.toggleMode)
QObject.connect(self.actionCommodity_Editor, SIGNAL('triggered()'), self.openEditor)
self.error_close = False
#set up required items for nn
self.training_image_dir = unicode(self.settings.app_path.decode('windows-1252'))+os.sep+u"nn_training_images"+os.sep
self.loadPlugins()
self.restorePos()
self.eddnthread = EDDNExport(self)
QObject.connect(self.eddnthread, SIGNAL('finished(QString)'), self.export.eddnFinished)
QObject.connect(self.eddnthread, SIGNAL('update(int,int)'), self.export.eddnUpdate)
self.thread = Worker()
self.connect(self.thread, SIGNAL("output(QString, QString)"), self.showUpdateAvailable)
self.thread.check(self.appversion)
if not self.settings.reg.contains('info_accepted'):
self.infoDialog = InfoDialog()
self.infoDialog.exec_()
else:
if not self.settings['info_accepted']:
self.infoDialog = InfoDialog()
self.infoDialog.exec_()
self.checkAppConfigXML()
def checkAppConfigXML(self):
path = unicode(self.settings['log_dir']).encode('windows-1252')+"\\..\\AppConfig.xml"
if isfile(path):
file = codecs.open(path, 'r', "utf-8")
file_content = file.read()
file.close()
start = file_content.find("<Network")
end = file_content.find("</Network>")
position = file_content.lower().find('verboselogging="1"', start, end)
if position == -1:
msg = _translate("EliteOCR","You don't have \"Verbose Logging\" enabled in your AppConfig.xml. It is necessary for automatic system name recognition. Do you want EliteOCR to enable it for you?", None)
reply = QMessageBox.question(self, 'Change File', msg, _translate("EliteOCR","Yes", None), _translate("EliteOCR","No", None))
if reply == 0:
file = codecs.open(unicode(self.settings['log_dir']).encode('windows-1252')+"\\..\\AppConfig_backup.xml", 'w', "utf-8")
file.write(file_content)
file.close()
newfile = file_content[:start+8] + '\n VerboseLogging="1"' + file_content[start+8:]
file = codecs.open(path, 'w', "utf-8")
file.write(newfile)
file.close()
QMessageBox.information(self,"Restart the Game", "Please restart the game to apply the change in AppConfig.xml")
else:
return
def resizeElements(self):
fields = [self.system_name, self.station_name, self.name, self.sell, self.buy, self.demand_num, self.demand, self.supply_num, self.supply, self.label_12, self.file_label, self.system_not_found]
for field in fields:
field.setMinimumSize(QSize(0, self.settings['input_size']))
field.setMaximumSize(QSize(16777215, self.settings['input_size']))
canvases = [self.station_name_img, self.name_img, self.sell_img, self.buy_img, self.demand_img, self.demand_text_img, self.supply_img, self.supply_text_img]
for canvas in canvases:
canvas.setMinimumSize(QSize(0, self.settings['snippet_size']))
canvas.setMaximumSize(QSize(16777215, self.settings['snippet_size']))
def genDarkStyle(self):
style = """
QWidget {{ background-color: #000; font-size: 10pt; font-family: Consolas}}
QLabel {{ color: {0};}}
QCheckBox {{ color: {0}; }}
QPushButton {{color: {2};
background-color: #000;
border: 1px solid {3};
min-height: 13px;
padding: 2px;}}
QToolButton {{color: {2};
background-color: #000;
border: 1px solid {3};
min-height: 12px;
padding: 2px;}}
QPushButton[enabled="false"]{{color: #555; border: 1px solid #555;}}
QToolButton[enabled="false"]{{color: #555; border: 1px solid #555;}}
QStatusBar {{ background-color: #000; color: {0};}}
QMenuBar {{ background-color: #000; color: {0};}}
QMenuBar::item {{ background-color: #000; color: {0};}}
QMenuBar::item:selected {{ background-color: #888; color: {0};}}
QMenu {{ background-color: #000; color: {0};}}
QMenu::item {{ background-color: #000; color: {0};}}
QMenu::item:selected {{ background-color: #888; color: {0};}}
QListWidget {{ background-color: #000; min-width: 150px}}
QFrame[frameShape="4"] {{ background-color: #888; }}
QFrame[frameShape="5"] {{ background-color: #888; }}
QGraphicsView {{ background-color: #000; border: 1px solid {4}}}
QTableWidget {{ background-color: #000; color: {0}; border: 1px solid {4}}}
QLineEdit {{ background-color: #000; border: 1px solid {4}; color: {1}; font-size: 13pt; font-family: Consolas}}
QComboBox {{ background-color: #000; border: 1px solid {4}; color: {1};}}
QComboBox:editable {{color: {1}; font-size: 11pt}}
QComboBox::down-arrow {{ image: url(:/ico/arrow.png); }}
QComboBox::drop-down:editable {{color: {1};}}
QHeaderView::section {{ background-color: #000; color: {0}; border: 1px solid {4}; padding: 2px; }}
QTableView QTableCornerButton::section {{ background: #000;}}
QSplitter {{ background-color: #0a0; color: #a00; }}
QProgressBar {{ border: 1px solid {4}; background-color: #000;}}
QProgressBar::chunk {{ background-color: #0a0; width: 20px; }}
QDoubleSpinBox {{ background-color: #888;}}
QSpinBox {{ background-color: #888;}}
QWebView {{ background-color: #888;}}
QTreeView {{ color: {0}; border: 1px solid {4}}}
QTabBar::tab {{ background-color: #000; color:{0}; border: 1px solid {4}; padding: 4px;}}
QListView {{ color: {1}; border: 1px solid {4}}}
""".format(str(self.settings['label_color']),str(self.settings['input_color']),str(self.settings['button_color']),str(self.settings['button_border_color']),str(self.settings['border_color']))
return style
def showUpdateAvailable(self, dir, appversion):
self.newupd = (dir, appversion)
self.statusbar.showMessage(unicode(_translate("EliteOCR","New version of EliteOCR available: %s To download it go to Help > Update", None)) % appversion, 0)
def restorePos(self):
self.settings.reg.beginGroup("MainWindow")
self.resize(self.settings.reg.value("size", QSize(400, 400)).toSize())
self.move(self.settings.reg.value("pos", QPoint(200, 200)).toPoint())
if self.settings.reg.value("maximized", False, type=bool):
self.showMaximized()
self.settings.reg.endGroup()
def closeEvent(self, event):
self.settings.reg.beginGroup("MainWindow")
self.settings.reg.setValue("size", self.size())
self.settings.reg.setValue("pos", self.pos())
if self.windowState() == Qt.WindowMaximized:
self.settings.reg.setValue("maximized", True)
else:
self.settings.reg.setValue("maximized", False)
self.settings.reg.endGroup()
self.settings.reg.setValue("public_mode", self.actionPublic_Mode.isChecked())
self.settings.reg.setValue("zoom_factor", self.factor.value())
self.settings.reg.sync()
if not self.busyDialog is None:
self.busyDialog.close()
event.accept()
def toggleMode(self):
if self.actionPublic_Mode.isChecked():
msg = _translate("EliteOCR","Switching to public mode will clear the result table! Are you sure you want to do it?", None)
reply = QMessageBox.question(self, 'Mode', msg, _translate("EliteOCR","Yes", None), _translate("EliteOCR","No", None))
if reply == 0:
self.clearTable()
self.cleanAllFields()
self.cleanAllSnippets()
else:
self.actionPublic_Mode.setChecked(False)
else:
msg = _translate("EliteOCR","Switching to private mode will disable BPC and EDDN Export! Are you sure you want to do it?", None)
reply = QMessageBox.question(self, 'Mode', msg, QMessageBox.Yes, QMessageBox.No)
if reply == QMessageBox.Yes:
self.enableButton(self.bpc_button, False)
self.enableButton(self.eddn_button,False)
else:
self.actionPublic_Mode.setChecked(True)
def loadPlugins(self):
"""Load known plugins"""
#Trade Dangerous Export by gazelle (bgol)
if isfile(self.settings.app_path+os.sep+"plugins"+os.sep+"TD_Export"+os.sep+"TD_Export.py"):
plugin2 = imp.load_source('TD_Export', self.settings.app_path+\
os.sep+"plugins"+os.sep+"TD_Export"+os.sep+"TD_Export.py")
self.tdexport = plugin2.TD_Export(self, self.settings.app_path.decode('windows-1252'))
self.tdexport_button = QPushButton(self.centralwidget)
self.tdexport_button.setText("Trade Dangerous Export")
self.enableButton(self.tdexport_button, False)
self.horizontalLayout_4.addWidget(self.tdexport_button)
self.tdexport_button.clicked.connect(lambda: self.tdexport.run(self.export.tableToList(False, True)))
def enablePluginButtons(self):
if 'tdexport' in dir(self):
if self.tdexport != None:
self.enableButton(self.tdexport_button, True)
def disablePluginButtons(self):
if 'tdexport' in dir(self):
if self.tdexport != None:
self.enableButton(self.tdexport_button, False)
def About(self):
QMessageBox.about(self,"About", "EliteOCR\nVersion "+self.appversion+"\n\n"+\
"Contributors:\n"+\
"Seeebek, CapCap, Gazelle, GMib, Ph.Baumann\n\n"+\
"EliteOCR is capable of reading the entries in Elite: Dangerous markets screenshots.\n\n")
def setupTable(self):
"""Add columns and column names to the table"""
"""
"self.result_table.setColumnCount(11)
self.result_table.setHorizontalHeaderLabels(['station', 'commodity', 'sell', 'buy',
'demand', 'dem', 'supply',
'sup', 'timestamp','system','img_height'])
"""
self.result_table.setColumnHidden(8, True)
self.result_table.setColumnHidden(10, True)
#self.result_table.setColumnHidden(11, True)
pass
def openHelp(self):
self.helpDialog = HelpDialog(self.settings.app_path.decode('windows-1252'))
self.helpDialog.setModal(False)
self.helpDialog.show()
def openUpdate(self):
self.updateDialog = UpdateDialog(self.settings.app_path.decode('windows-1252'), self.appversion, self.newupd)
self.updateDialog.setModal(False)
self.updateDialog.show()
def openSettings(self):
"""Open settings dialog and reload settings"""
settingsDialog = SettingsDialog(self.settings)
settingsDialog.exec_()
if self.settings["theme"] == "dark":
self.dark_theme = True
self.darkstyle = self.genDarkStyle()
self.style = self.darkstyle
else:
self.dark_theme = False
self.style = self.def_style
#self.app.setStyleSheet("")
self.app.setStyleSheet(self.style)
self.resizeElements()
def openEditor(self):
editorDialog = EditorDialog(self.settings)
editorDialog.exec_()
def addAllScreenshots(self):
dir = unicode(self.settings['screenshot_dir']).encode('windows-1252')
#gen = (join(dir, file).decode('windows-1252') for file in listdir(dir) if isfile(join(dir, file)))
gen = [join(dir, file).decode('windows-1252') for file in listdir(dir) if file.endswith('.bmp')]
files = []
for file in gen:
files.append(file)
self.addFiles(files)
def addFiles(self, screenshots = None):
"""Add files to the file list."""
if screenshots:
files = screenshots
else:
if self.settings["native_dialog"]:
files = QFileDialog.getOpenFileNames(self, "Open", self.settings['screenshot_dir'])
else:
files = QFileDialog.getOpenFileNames(self, "Open", self.settings['screenshot_dir'], options = QFileDialog.DontUseNativeDialog)
if files == []:
return
first_item = None
self.progress_bar.setMinimum(0)
self.progress_bar.setMaximum(len(files))
self.progress_bar.setValue(0)
counter = 0
for file in files:
file1 = unicode(file).encode('windows-1252')
item = CustomQListWidgetItem(split(file1)[1], file1, self.settings)
if first_item == None:
first_item = item
self.file_list.addItem(item)
counter+=1
self.progress_bar.setValue(counter)
self.progress_bar.setValue(0)
self.file_list.itemClicked.connect(self.selectFile)
self.save_button.setEnabled(False)
self.enableButton(self.skip_button, False)
#self.cleanAllFields()
#self.cleanAllSnippets()
if first_item !=None:
self.selectFile(first_item)
if self.ocr_button.isEnabled() and self.file_list.count() > 1:
self.enableButton(self.ocr_all, True)
self.cleanAllFields()
self.cleanAllSnippets()
def removeAllFiles(self):
files = self.file_list.count()
for i in xrange(files):
item = self.file_list.currentItem()
self.file_list.takeItem(self.file_list.currentRow())
del item
self.file_label.setText("-")
scene = QGraphicsScene()
self.previewSetScene(scene)
self.save_button.setEnabled(False)
self.skip_button.setEnabled(False)
self.cleanAllFields()
self.cleanAllSnippets()
self.remove_button.setEnabled(False)
self.remove_all_button.setEnabled(False)
self.ocr_button.setEnabled(False)
self.enableButton(self.zoom_button, False)
def softRemoveFile(self):
item = self.file_list.currentItem()
self.file_list.takeItem(self.file_list.currentRow())
del item
def removeFile(self):
"""Remove selected file from file list."""
item = self.file_list.currentItem()
self.file_list.takeItem(self.file_list.currentRow())
del item
self.file_label.setText("-")
scene = QGraphicsScene()
self.previewSetScene(scene)
self.save_button.setEnabled(False)
self.enableButton(self.skip_button, False)
self.cleanAllFields()
self.cleanAllSnippets()
if self.file_list.currentItem():
self.selectFile(self.file_list.currentItem())
if self.file_list.count() == 0:
self.remove_button.setEnabled(False)
self.remove_all_button.setEnabled(False)
self.ocr_button.setEnabled(False)
self.enableButton(self.zoom_button, False)
if self.file_list.count() < 2:
self.enableButton(self.ocr_all, False)
def selectFile(self, item):
"""Select clicked file and shows prewiev of the selected file."""
self.cleanAllFields()
self.cleanAllSnippets()
self.progress_bar.setMinimum(0)
self.progress_bar.setMaximum(20)
self.progress_bar.setValue(0)
self.color_image = item.loadColorImage()
self.progress_bar.setValue(10)
self.preview_image = item.loadPreviewImage(self.color_image, self)
self.progress_bar.setValue(20)
self.ocr_all_set = False
#font = QFont("Consolas", 11)
self.system_not_found.setText("")
if len(item.system) == 0:
self.system_not_found.setText(_translate("EliteOCR","System name not found in log files. Please read Help for more info.", None))
self.system_name.setText(item.system)
if not item.station is None:
self.station_name.setText(item.station)
#self.system_name.setFont(font)
self.file_list.setCurrentItem(item)
self.file_label.setText(item.text())
self.setPreviewImage(self.preview_image)
self.remove_button.setEnabled(True)
self.remove_all_button.setEnabled(True)
self.enableButton(self.continue_button, False)
if not item.valid_market:
self.system_not_found.setText(_translate("EliteOCR","File was not recognized as a valid market screenshot. Please read Help for more info.", None))
self.progress_bar.setValue(0)
return
self.ocr_button.setEnabled(True)
self.enableButton(self.zoom_button, True)
if self.file_list.count() > 1:
self.enableButton(self.ocr_all, True)
self.progress_bar.setValue(0)
def setPreviewImage(self, image):
"""Show image in self.preview."""
factor = self.factor.value()
pix = image.scaled(QSize(self.preview.size().width()*factor,self.preview.size().height()*factor), Qt.KeepAspectRatio, Qt.SmoothTransformation)
scene = QGraphicsScene()
scene.addPixmap(pix)
self.previewSetScene(scene)
def previewSetScene(self, scene):
"""Shows scene in preview"""
self.preview.setScene(scene)
self.preview.show()
def runOCRAll(self):
self.ocr_all_set = True
self.performOCR()
def performOCR(self):
"""Send image to OCR and process the results"""
self.OCRline = 0
self.busyDialog = BusyDialog(self)
self.busyDialog.show()
QApplication.processEvents()
if self.file_list.currentItem().valid_market:
self.current_result = OCR(self, self.color_image, self.file_list.currentItem().ocr_areas, self.settings["ocr_language"], self.file_list.currentItem())
self.busyDialog.close()
"""
try:
self.current_result = OCR(self.color_image)
except:
QMessageBox.critical(self,"Error", "Error while performing OCR.\nPlease report the "+\
"problem to the developers through github, sourceforge or forum and provide the "+\
"screenshot which causes the problem.")
return
if self.current_result.station == None:
QMessageBox.critical(self,"Error", "Screenshot not recognized.\n"+\
"Make sure you use a valid screenshot from the commodieties market. Should the "+\
"problem persist, please recalibrate the OCR areas with Settings->Calibrate.")
return
"""
if len(self.current_result.commodities) < 1:
QMessageBox.critical(self,"Error", "No results found!\nYou might be using an unsupported HUD color. Please read help for more information.")
return
self.drawOCRPreview()
self.markCurrentRectangle()
self.drawStationName()
self.skip_button.setEnabled(True)
self.enableButton(self.save_button, True)
self.processOCRLine()
self.system_name.setFocus()
if self.settings['create_nn_images']:
self.saveStationForTraining()
else:
self.nextFile()
def addItemToTable(self):
"""Adds items from current OCR result line to the result table."""
tab = self.result_table
res_station = unicode(self.station_name.text()).title()
row_count = tab.rowCount()
self.export_button.setEnabled(True)
if self.actionPublic_Mode.isChecked():
self.bpc_button.setEnabled(True)
self.eddn_button.setEnabled(True)
self.enablePluginButtons()
self.enableButton(self.clear_table, True)
# check if no stock and not purchased
if self.demand_num.text() == "" and self.supply_num.text() == "":
self.nextLine()
return
#check for duplicates
duplicate = False
if self.settings["remove_dupli"]:
for i in range(row_count):
station = unicode(tab.item(i, 0).text()).title()
com1 = unicode(tab.item(i, 1).text()).title()
com2 = unicode(self.fields[0].text()).replace(',', '').title()
if station == res_station and com1 == com2:
duplicate = True
if not duplicate:
self.current_result.station.name.value = self.station_name.text()
tab.insertRow(row_count)
newitem = QTableWidgetItem(unicode(res_station).title().replace("'S", "'s"))
tab.setItem(row_count, 0, newitem)
for n, field in enumerate(self.fields):
newitem = QTableWidgetItem(unicode(field.text()).replace(',', '').title())
tab.setItem(row_count, n+1, newitem)
newitem = QTableWidgetItem(self.file_list.currentItem().timestamp)
tab.setItem(row_count, 8, newitem)
newitem = QTableWidgetItem(self.system_name.text())
tab.setItem(row_count, 9, newitem)
newitem = QTableWidgetItem(unicode(self.file_list.currentItem().market_width))
tab.setItem(row_count, 10, newitem)
tab.resizeColumnsToContents()
tab.resizeRowsToContents()
if self.settings['create_nn_images']:
self.saveValuesForTraining()
self.nextLine()
def saveValuesForTraining(self):
"""Get OCR image/user values and save them away for later processing, and training
neural net"""
cres = self.current_result
res = cres.commodities[self.OCRline]
if not exists(self.training_image_dir):
makedirs(self.training_image_dir)
if not exists(self.training_image_dir+os.sep+"text"):
makedirs(self.training_image_dir+os.sep+"text")
if not exists(self.training_image_dir+os.sep+"numbers"):
makedirs(self.training_image_dir+os.sep+"numbers")
w = len(self.current_result.contrast_commodities_img)
h = len(self.current_result.contrast_commodities_img[0])
for index, field, canvas, item in zip(range(0, len(self.canvases) - 1),
self.fields, self.canvases, res.items):
val = unicode(field.text())#.replace(',', '')
if field in [self.sell, self.buy, self.demand_num, self.supply_num]:
if val:
snippet = self.cutImage(cres.contrast_commodities_img, item)
#cv2.imshow('snippet', snippet)
imageFilepath = self.training_image_dir + os.sep + u'numbers' + os.sep + unicode(val) + u'_' + unicode(w) + u'x' + unicode(h) +\
u'-' + unicode(int(time())) + u'-' +\
unicode(random.randint(10000, 100000)) + u'.png'
cv2.imwrite(imageFilepath.encode('windows-1252'), snippet)
elif field in [self.name]:
if val:
snippet = self.cutImage(cres.contrast_commodities_img, item)
#cv2.imshow('snippet', snippet)
imageFilepath = self.training_image_dir + os.sep + u'text' + os.sep + unicode(val) + u'_' + unicode(w) + u'x' + unicode(h) +\
u'-' + unicode(int(time())) + u'-' +\
unicode(random.randint(10000, 100000)) + u'.png'
cv2.imwrite(imageFilepath.encode('windows-1252'), snippet)
def saveStationForTraining(self):
cres = self.current_result
if not exists(self.training_image_dir):
makedirs(self.training_image_dir)
if not exists(self.training_image_dir+os.sep+"text"):
makedirs(self.training_image_dir+os.sep+"text")
w = len(self.current_result.contrast_commodities_img)
h = len(self.current_result.contrast_commodities_img[0])
snippet = self.cutImage(cres.contrast_station_img, cres.station.name)
val = self.station_name.text()
imageFilepath = self.training_image_dir + os.sep + u'text' + os.sep + unicode(val) + u'_' + unicode(w) + u'x' + unicode(h) +\
u'-' + unicode(int(time())) + u'-' +\
unicode(random.randint(10000, 100000)) + u'.png'
cv2.imwrite(imageFilepath.encode('windows-1252'), snippet)
def cutImageForTraining(self, image, item):
"""Cut image snippet from a big image using points from item."""
snippet = image[item.y1:item.y2,
item.x1:item.x2]
return snippet
def continueOCR(self):
if self.ocr_all_set:
self.nextFile()
else:
if self.settings['delete_files']:
print "Deleted (continueOCR 458):"
print self.file_list.currentItem().text()
remove(self.file_list.currentItem().hiddentext)
self.removeFile()
self.enableButton(self.continue_button, False)
def nextLine(self):
"""Process next OCR result line."""
self.markCurrentRectangle(QPen(Qt.green))
self.OCRline += 1
if len(self.previewRects) > self.OCRline:
self.markCurrentRectangle()
self.processOCRLine()
self.name.setFocus()
else:
self.save_button.setEnabled(False)
self.enableButton(self.skip_button, False)
self.cleanAllFields()
self.cleanAllSnippets()
if self.ocr_all_set:
if self.settings['pause_at_end']:
self.enableButton(self.continue_button, True)
else:
self.nextFile()
else:
if self.settings['delete_files']:
if self.settings['pause_at_end']:
self.enableButton(self.continue_button, True)
else:
print "Deleted (nextLine 486):"
print self.file_list.currentItem().text()
remove(self.file_list.currentItem().hiddentext)
self.removeFile()
def nextFile(self):
"""OCR next file"""
if self.file_list.currentRow() < self.file_list.count()-1:
if self.settings['delete_files']:
print "Deleted (nextFile 496):"
print self.file_list.currentItem().text()
remove(self.file_list.currentItem().hiddentext)
self.softRemoveFile()
else:
self.file_list.setCurrentRow(self.file_list.currentRow() + 1)
self.color_image = self.file_list.currentItem().loadColorImage()
self.preview_image = self.file_list.currentItem().loadPreviewImage(self.color_image, self)
self.performOCR()
#font = QFont("Consolas", 11)
if self.OCRline == 0:
if len(self.file_list.currentItem().system) > 0:
self.system_not_found.setText("")
self.system_name.setText(self.file_list.currentItem().system)
if not self.file_list.currentItem().station is None:
self.station_name.setText(self.file_list.currentItem().station)
#self.system_name.setFont(font)
else:
self.system_name.setText("")
#self.system_name.setFont(font)
self.system_not_found.setText(_translate("EliteOCR","System name not found in log files. Make sure log directory path is set up correctly or add system name manually in the field below. Note: System name is necessary for BPC import!",None))
#self.system_name.setFocus()
#self.system_name.selectAll()
else:
if self.settings['delete_files']:
print "Deleted (nextFile 520):"
print self.file_list.currentItem().text()
remove(self.file_list.currentItem().hiddentext)
self.softRemoveFile()
def clearTable(self):
"""Empty the result table."""
self.result_table.setRowCount(0)
self.clear_table.setEnabled(False)
self.export_button.setEnabled(False)
self.bpc_button.setEnabled(False)
self.enableButton(self.eddn_button, False)
self.disablePluginButtons()
def processOCRLine(self):
"""Process current OCR result line."""
if len(self.current_result.commodities) > self.OCRline:
#font = QFont("Consolas", 11)
res = self.current_result.commodities[self.OCRline]
if self.OCRline > 0:
autofill = True
else:
autofill = False
if self.settings["auto_fill"]:
for item in res.items:
if item == None:
continue
if not item.confidence > 0.83:
autofill = False
if res.items[0] is None:
autofill = False
if res.items[1] is None:
autofill = False
if self.file_list.currentItem().market_width < 1065 and self.actionPublic_Mode.isChecked():
autofill = False
self.save_button.setEnabled(False)
self.enableButton(self.skip_button, True)
QTimer.singleShot(1200, partial(self.enableButton, self.save_button, True));
#QTimer.singleShot(1500, partial(self.skip_button.setEnabled, True));
for field, canvas, item in zip(self.fields, self.canvases, res.items):
if item != None:
#field.clear()
#field.addItems(item.optional_values)
field.setText(item.value)
#field.lineEdit().setFont(font)
if not(self.settings["auto_fill"] and autofill):
self.setConfidenceColor(field, item)
self.drawSnippet(canvas, item)
else:
self.cleanSnippet(canvas)
self.cleanField(field)
if self.settings["auto_fill"] and autofill:
self.addItemToTable()
def setConfidenceColor(self, field, item):
c = item.confidence
#c = random.random()
if self.dark_theme:
if c > 0.83:
color = "#000"
if c <= 0.83 and c >0.67:
color = "#666600"
if c <= 0.67 and c >0.5:
color = "#665100"
if c <= 0.5 and c >0.34:
color = "#663e00"
if c <= 0.34 and c >0.17:
color = "#662900"
if c <= 0.17:
color = "#661500"
else:
if c > 0.83:
color = "#ffffff"
if c <= 0.83 and c >0.67:
color = "#ffffbf"
if c <= 0.67 and c >0.5:
color = "#fff2bf"
if c <= 0.5 and c >0.34:
color = "#ffe6bf"
if c <= 0.34 and c >0.17:
color = "#ffd9bf"
if c <= 0.17:
color = "#ffccbf"
field.setStyleSheet("background: "+color+";")
def drawOCRPreview(self):
if self.current_result is None:
self.setPreviewImage(self.preview_image)
return
factor = self.factor.value()
res = self.current_result
name = res.station
img = self.preview_image
old_h = img.height()
old_w = img.width()
pix = img.scaled(QSize(self.preview.size().width()*factor,self.preview.size().height()*factor), Qt.KeepAspectRatio, Qt.SmoothTransformation)
#pix = img.scaled(self.preview.size(), Qt.KeepAspectRatio, Qt.SmoothTransformation)
new_h = pix.height()
new_w = pix.width()
ratio_h = old_h/float(new_h)
ratio_w = old_w/float(new_w)
self.scene = QGraphicsScene()
self.scene.addPixmap(pix)
#self.scene.addPixmap(img)
self.previewRects = []
pen = QPen(Qt.yellow)
redpen = QPen(Qt.red)
bluepen = QPen(Qt.blue)
greenpen = QPen(Qt.green)
rect = self.addRect(self.scene, name, ratio_w, ratio_h, greenpen)
counter = 0
for line in res.commodities:
if counter < self.OCRline:
rect = self.addRect(self.scene, line, ratio_w, ratio_h, greenpen)
elif counter == self.OCRline:
rect = self.addRect(self.scene, line, ratio_w, ratio_h, bluepen)
else:
if line.w < (0.02*old_w):
rect = self.addRect(self.scene, line, ratio_w, ratio_h, redpen)
else:
rect = self.addRect(self.scene, line, ratio_w, ratio_h, pen)
counter += 1
self.previewRects.append(rect)
self.previewSetScene(self.scene)
def addRect(self, scene, item, ratio_w, ratio_h, pen):
"""Adds a rectangle to scene and returns it."""
rect = scene.addRect(item.x1/ratio_w -3, item.y1/ratio_h -3,
item.w/ratio_w +7, item.h/ratio_h +6, pen)
return rect
def markCurrentRectangle(self, pen=QPen(Qt.blue)):
self.previewRects[self.OCRline].setPen(pen)
def cutImage(self, image, item):
"""Cut image snippet from a big image using points from item."""
snippet = image[item.y1 - 5:item.y2 + 5,
item.x1 - 5:item.x2 + 5]
return snippet
def drawSnippet(self, graphicsview, item):
"""Draw single result item to graphicsview"""
res = self.current_result
snippet = self.cutImage(res.contrast_commodities_img, item)
if self.dark_theme:
snippet = 255 - snippet
#cv2.imwrite('snippets/'+unicode(self.currentsnippet)+'.png',snippet)
#self.currentsnippet += 1
processedimage = array2qimage(snippet)
pix = QPixmap()
pix.convertFromImage(processedimage)
pix = pix.scaled(graphicsview.width(), graphicsview.height(), Qt.KeepAspectRatio, Qt.SmoothTransformation)
scene = QGraphicsScene()
scene.addPixmap(pix)
graphicsview.setScene(scene)
graphicsview.show()
def drawStationName(self):
"""Draw station name snippet to station_name_img"""
res = self.current_result
name = res.station.name
#self.station_name.setText('')
#self.station_name.clear()
#self.station_name.addItems(name.optional_values)
if not self.file_list.currentItem().station is None:
self.station_name.setText(self.file_list.currentItem().station)
else:
self.station_name.setText(name.value)
#font = QFont("Consolas", 11)
#self.station_name.lineEdit().setFont(font)
#self.setConfidenceColor(self.station_name, name)
img = self.cutImage(res.contrast_station_img, name)
if self.dark_theme:
img = 255 - img
processedimage = array2qimage(img)
pix = QPixmap()
pix.convertFromImage(processedimage)
if self.station_name_img.height() < pix.height():
pix = pix.scaled(self.station_name_img.size(),
Qt.KeepAspectRatio,
Qt.SmoothTransformation)
scene = QGraphicsScene()
scene.addPixmap(pix)
self.station_name_img.setScene(scene)
self.station_name_img.show()
def cleanAllFields(self):
for field in self.fields:
self.cleanField(field)
self.cleanField(self.station_name)
self.save_button.setEnabled(False)
self.enableButton(self.skip_button, False)
def cleanField(self, field):
field.setText('')
if self.dark_theme:
field.setStyleSheet("background: #000;")
else:
field.setStyleSheet("background: #fff;")
#field.lineEdit().setStyleSheet("")
#field.clear()
def cleanAllSnippets(self):
for field in self.canvases:
self.cleanSnippet(field)
self.cleanSnippet(self.station_name_img)
def cleanSnippet(self, graphicsview):
scene = QGraphicsScene()
graphicsview.setScene(scene)
def enableButton(self, button, switch):
button.setEnabled(switch)
if self.dark_theme:
self.app.setStyleSheet("")
self.app.setStyleSheet(self.style)
self.repaint()
