def main(): """ NAME upload_magic.py DESCRIPTION This program will prepare your PMAG text files created by the programs nfo_magic.py, zeq_magic.py, thellier_magic.py, mag_magic, specimens_results_magic.py and so on. it will check for all the MagIC text files and skip the missing ones SYNTAX upload_magic.py INPUT MagIC txt files OPTIONS -h prints help message and quits -all include all the measurement data, default is only those used in interpretations OUTPUT upload.txt: file for uploading to MagIC database """ if '-h' in sys.argv: print main.__doc__ sys.exit() else: dataframe = extractor.command_line_dataframe([['cat', False, 0], ['F', False, ''], ['f', False, '']]) checked_args = extractor.extract_and_check_args(sys.argv, dataframe) dir_path, concat = extractor.get_vars(['WD', 'cat'], checked_args) ipmag.upload_magic(concat, dir_path)
def test_empty_dir(self): directory = os.path.join(self.dir_path, 'empty_dir') outfile, error_message, errors = ipmag.upload_magic(dir_path=directory) self.assertFalse(errors) self.assertFalse(outfile) self.assertEqual(error_message, "no data found, upload file not created")
def on_btn_upload(self, event): outstring = "upload_magic.py" print "-I- running python script:\n %s" % (outstring) wait = wx.BusyInfo("Please wait, working...") wx.Yield() upfile, error_message, errors = ipmag.upload_magic( dir_path=self.WD, data_model=self.er_magic.data_model) del wait if upfile: text = "You are ready to upload.\n Your file: {} was generated in MagIC Project Directory.\nDrag and drop this file in the MagIC database.".format( os.path.split(upfile)[1]) dlg = wx.MessageDialog(self, caption="Saved", message=text, style=wx.OK) else: text = "There were some problems with the creation of your upload file.\nError message: {}\nSee Terminal/Command Prompt for details".format( error_message) dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy()
def test_empty_dir(self): directory = os.path.join(self.dir_path, 'empty_dir') outfile, error_message, errors = ipmag.upload_magic(dir_path=directory) self.assertFalse(errors) self.assertFalse(outfile) self.assertEqual(error_message, "no data found, upload file not created") files = os.listdir(directory) self.assertEqual(['blank.txt'], files)
def on_btn_upload(self, event): outstring = "upload_magic.py" print("-I- running python script:\n %s" % (outstring)) wait = wx.BusyInfo("Please wait, working...") wx.Yield() if self.data_model_num == 3: res, error_message, has_problems, all_failing_items = ipmag.upload_magic3( dir_path=self.WD, vocab=self.contribution.vocab, contribution=self.contribution) if self.data_model_num == 2: res, error_message, errors = ipmag.upload_magic( dir_path=self.WD, data_model=self.er_magic.data_model) del wait if res: text = "You are ready to upload.\n Your file: {} was generated in MagIC Project Directory.\nDrag and drop this file in the MagIC database.".format( os.path.split(res)[1]) dlg = wx.MessageDialog(self, caption="Saved", message=text, style=wx.OK) else: text = "There were some problems with the creation of your upload file.\nError message: {}\nSee Terminal/message window for details".format( error_message) dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy() if self.data_model_num == 3: from programs import magic_gui self.Disable() self.Hide() self.magic_gui_frame = magic_gui.MainFrame( self.WD, dmodel=self.data_model, title="Validations", contribution=self.contribution) self.magic_gui_frame.validation_mode = ['specimens'] self.magic_gui_frame.failing_items = all_failing_items self.magic_gui_frame.change_dir_button.Disable() self.magic_gui_frame.Centre() self.magic_gui_frame.Show() self.magic_gui_frame.highlight_problems(has_problems) # # change name of upload button to 'exit validation mode' self.magic_gui_frame.bSizer2.GetStaticBox().SetLabel( 'return to main GUI') self.magic_gui_frame.btn_upload.SetLabel("exit validation mode") # bind that button to quitting magic gui and re-enabling Pmag GUI self.magic_gui_frame.Bind(wx.EVT_BUTTON, self.on_end_validation, self.magic_gui_frame.btn_upload)
def test_with_valid_files(self): #print os.path.join(self.dir_path, 'my_project') outfile, error_message, errors = ipmag.upload_magic(dir_path=os.path.join(self.dir_path, 'my_project')) self.assertTrue(outfile) self.assertEqual(error_message, '') self.assertFalse(errors) assert os.path.isfile(outfile) directory = os.path.join(self.dir_path, 'my_project_with_errors') os.remove(os.path.join(directory, outfile))
def main(): """ NAME upload_magic.py DESCRIPTION This program will prepare your MagIC text files for uploading to the MagIC database it will check for all the MagIC text files and skip the missing ones SYNTAX upload_magic.py INPUT MagIC txt files OPTIONS -h prints help message and quits -all include all the measurement data, default is only those used in interpretations -DM specify which MagIC data model number to use (2 or 3). Default is 3. OUTPUT upload file: file for uploading to MagIC database """ if '-h' in sys.argv: print(main.__doc__) sys.exit() else: data_model_num = pmag.get_named_arg("-DM", 3) dataframe = extractor.command_line_dataframe([['cat', False, 0], ['F', False, ''], ['f', False, '']]) checked_args = extractor.extract_and_check_args(sys.argv, dataframe) dir_path, concat = extractor.get_vars(['WD', 'cat'], checked_args) data_model_num = int(float(data_model_num)) if data_model_num == 2: ipmag.upload_magic2(concat, dir_path) else: ipmag.upload_magic(concat, dir_path)
def test3_with_contribution(self): dir_path = os.path.join(WD, 'data_files', '3_0', 'Megiddo') con = cb.Contribution(directory=dir_path) outfile, error_message, errors, all_errors = ipmag.upload_magic(contribution=con) msg = "Validation of your upload file has failed.\nYou can still upload" self.assertTrue(error_message.startswith(msg)) # delete any upload file that was partially created import re pattern = re.compile('\A[^.]*\.[a-zA-Z]*\.\d{4}\_?\d*\.txt') possible_files = os.listdir(dir_path) files = [] for f in possible_files: if pattern.match(f): files.append(f) pmag.remove_files(files, dir_path)
def test_depth_propagation(self): dir_path = os.path.join(WD, 'data_files', 'core_depthplot') #con = cb.Contribution(dir_path) #self.assertNotIn('core_depth', con.tables['sites'].df.index) #con.propagate_cols(['core_depth'], 'sites', 'samples', down=False) #self.assertIn('core_depth', con.tables['sites'].df.columns) #self.assertEqual(con.tables['sites'].df.loc['15-1-013', 'core_depth'], 55.23) # outfile, error_message, errors, all_errors = ipmag.upload_magic(dir_path=dir_path) print('mv {} {}'.format(outfile, WD)) os.system('mv {} {}'.format(outfile, WD)) outfile = os.path.join(WD, os.path.split(outfile)[1]) ipmag.download_magic(outfile) con = cb.Contribution(WD) self.assertIn('core_depth', con.tables['sites'].df.columns) self.assertEqual(con.tables['sites'].df.loc['15-1-013', 'core_depth'], 55.23)
def test_with_invalid_files(self): directory = os.path.join(self.dir_path, 'my_project_with_errors') outfile, error_message, errors = ipmag.upload_magic(dir_path=directory) self.assertTrue(errors) self.assertFalse(outfile) self.assertEqual(error_message, "file validation has failed. You may run into problems if you try to upload this file.") directory = os.path.join(self.dir_path, 'my_project_with_errors') # delete any upload file that was partially created import re pattern = re.compile('\w*[.]\w*[.]\w*[20]\d{2}\w*.txt$') possible_files = os.listdir(directory) files = [] for f in possible_files: if pattern.match(f): files.append(f) pmag.remove_files(files, directory)
def on_btn_upload(self, event): outstring="upload_magic.py" print("-I- running python script:\n %s"%(outstring)) wait = wx.BusyInfo("Please wait, working...") wx.Yield() if self.data_model_num == 3: res, error_message, has_problems, all_failing_items = ipmag.upload_magic3(dir_path=self.WD, vocab=self.contribution.vocab, contribution=self.contribution) if self.data_model_num == 2: res, error_message, errors = ipmag.upload_magic(dir_path=self.WD, data_model=self.er_magic.data_model) del wait if res: text = "You are ready to upload.\n Your file: {} was generated in MagIC Project Directory.\nDrag and drop this file in the MagIC database.".format(os.path.split(res)[1]) dlg = wx.MessageDialog(self, caption="Saved", message=text, style=wx.OK) else: text = "There were some problems with the creation of your upload file.\nError message: {}\nSee Terminal/message window for details".format(error_message) dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy() if self.data_model_num == 3: from programs import magic_gui self.Disable() self.Hide() self.magic_gui_frame = magic_gui.MainFrame(self.WD, dmodel=self.data_model, title="Validations", contribution=self.contribution) self.magic_gui_frame.validation_mode = ['specimens'] self.magic_gui_frame.failing_items = all_failing_items self.magic_gui_frame.change_dir_button.Disable() self.magic_gui_frame.Centre() self.magic_gui_frame.Show() self.magic_gui_frame.highlight_problems(has_problems) # # change name of upload button to 'exit validation mode' self.magic_gui_frame.bSizer2.GetStaticBox().SetLabel('return to main GUI') self.magic_gui_frame.btn_upload.SetLabel("exit validation mode") # bind that button to quitting magic gui and re-enabling Pmag GUI self.magic_gui_frame.Bind(wx.EVT_BUTTON, self.on_end_validation, self.magic_gui_frame.btn_upload)
def on_btn_upload(self, event): outstring="upload_magic.py" print "-I- running python script:\n %s"%(outstring) wait = wx.BusyInfo("Please wait, working...") wx.Yield() upfile, error_message, errors = ipmag.upload_magic(dir_path=self.WD, data_model=self.er_magic.data_model) del wait if upfile: text = "You are ready to upload.\n Your file: {} was generated in MagIC Project Directory.\nDrag and drop this file in the MagIC database.".format(os.path.split(upfile)[1]) dlg = wx.MessageDialog(self, caption="Saved", message=text, style=wx.OK) else: text = "There were some problems with the creation of your upload file.\nError message: {}\nSee Terminal/Command Prompt for details".format(error_message) dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy()
def on_upload_file(self, event): if not hasattr(self, "contribution"): self.contribution = cb.Contribution(self.WD, dmodel=self.data_model) wait = wx.BusyInfo('Validating data, please wait...') wx.SafeYield() res, error_message, has_problems, all_failing_items = ipmag.upload_magic(dir_path=self.WD, vocab=self.contribution.vocab) self.failing_items = all_failing_items if has_problems: self.highlight_problems(has_problems) if not has_problems: self.validation_mode = set() self.message.SetLabel('Validated!') self.bSizer_msg.ShowItems(False) self.hbox.Fit(self) # do alert that your file passed dlg = wx.MessageDialog(self,caption="Message:", message="Your contribution has passed validations!\nGo to https://www.earthref.org/MagIC to upload:\n{}".format(res), style=wx.OK) dlg.ShowModal() del wait
def on_btn_upload(self, event): """ Try to run upload_magic. Open validation mode if the upload file has problems. """ if not self.check_for_uncombined_files(): return outstring = "upload_magic.py" print("-I- running python script:\n %s" % (outstring)) wait = wx.BusyInfo("Please wait, working...") wx.SafeYield() success_responses = ['200', 200, '201', 201, True] if 'measurements' in self.contribution.tables: self.contribution.tables['measurements'].add_measurement_names() upload_file, val_response, dummy1, dummy2 = ipmag.upload_magic( concat=False, input_dir_path=self.WD, dir_path=self.WD) del wait if val_response == "no 3.0 files found, upload file not created": pw.simple_warning( "No 3.0 files were found in your directory, so no upload could be created!" ) return status = val_response['status'] if not status: pw.simple_warning( "Oops, something went wrong with validating on the server.\n{}\nTry again later or submit a bug report." .format(val_response['warnings'])) return validation_errors = val_response['validation'] if (not validation_errors['warnings']) and ( not validation_errors['errors']): text = "You are ready to upload!\n{} was generated in {}".format( os.path.split(upload_file)[1], self.WD) dlg = pw.ChooseOne(self, "Go to MagIC for uploading", "Not ready yet", text, "Saved") dlg.Centre() result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy() if result == wx.ID_YES: pw.on_database_upload(None) return # there were problems, so display validation text = "There were some problems with the creation of your upload file.\nSee Terminal/message window for details" dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) dlg.Centre() result = dlg.ShowModal() # TODO: get the error-y business formatted into a dict of lists of dicts from programs import magic_gui self.Disable() self.Hide() self.magic_gui_frame = magic_gui.MainFrame( self.WD, dmodel=self.data_model, title="Validations", contribution=self.contribution, errors=validation_errors['errors']) self.magic_gui_frame.Centre() self.magic_gui_frame.Show() self.magic_gui_frame.highlight_problems() # bind that button to quitting magic gui and re-enabling Pmag GUI self.magic_gui_frame.Bind(wx.EVT_BUTTON, self.on_end_validation, self.magic_gui_frame.return_btn) # do binding so that closing/quitting re-opens the main frame self.magic_gui_frame.Bind(wx.EVT_CLOSE, self.on_end_validation) # this makes it work with only the validation window open self.magic_gui_frame.Bind( wx.EVT_MENU, lambda event: self.menubar.on_quit(event, self.magic_gui_frame), self.magic_gui_frame.menubar.file_quit) # this makes it work if an additional grid is open self.Bind( wx.EVT_MENU, lambda event: self.menubar.on_quit(event, self.magic_gui_frame), self.magic_gui_frame.menubar.file_quit)
def on_btn_upload(self, event): """ Try to run upload_magic. Open validation mode if the upload file has problems. """ if not self.check_for_uncombined_files(): return outstring = "upload_magic.py" print("-I- running python script:\n %s" % (outstring)) wait = wx.BusyInfo("Please wait, working...") wx.SafeYield() self.contribution.tables['measurements'].add_measurement_names() if self.data_model_num == 3: res, error_message, has_problems, all_failing_items = ipmag.upload_magic3( dir_path=self.WD, vocab=self.contribution.vocab, contribution=self.contribution) if self.data_model_num == 2: res, error_message, errors = ipmag.upload_magic( dir_path=self.WD, data_model=self.er_magic.data_model) del wait if res: text = "You are ready to upload!\n{} was generated in {}".format( os.path.split(res)[1], os.path.split(res)[0]) dlg = pw.ChooseOne(self, "Go to MagIC for uploading", "Not ready yet", text, "Saved") del wait #dlg = wx.MessageDialog(self, caption="Saved", message=text, style=wx.OK) else: text = "There were some problems with the creation of your upload file.\nError message: {}\nSee Terminal/message window for details".format( error_message) dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) dlg.Centre() result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy() if result == wx.ID_YES: pw.on_database_upload(None) if self.data_model_num == 3: if not res: from programs import magic_gui self.Disable() self.Hide() self.magic_gui_frame = magic_gui.MainFrame( self.WD, dmodel=self.data_model, title="Validations", contribution=self.contribution) self.magic_gui_frame.validation_mode = ['specimens'] self.magic_gui_frame.failing_items = all_failing_items self.magic_gui_frame.change_dir_button.Disable() self.magic_gui_frame.Centre() self.magic_gui_frame.Show() self.magic_gui_frame.highlight_problems(has_problems) # # change name of upload button to 'exit validation mode' self.magic_gui_frame.bSizer2.GetStaticBox().SetLabel( 'return to main GUI') self.magic_gui_frame.btn_upload.SetLabel( "exit validation mode") # bind that button to quitting magic gui and re-enabling Pmag GUI self.magic_gui_frame.Bind(wx.EVT_BUTTON, self.on_end_validation, self.magic_gui_frame.btn_upload)
def on_btn_upload(self, event): """ Try to run upload_magic. Open validation mode if the upload file has problems. """ if not self.check_for_uncombined_files(): return outstring="upload_magic.py" print("-I- running python script:\n %s"%(outstring)) wait = wx.BusyInfo("Please wait, working...") wx.SafeYield() self.contribution.tables['measurements'].add_measurement_names() if self.data_model_num == 3: res, error_message, has_problems, all_failing_items = ipmag.upload_magic(concat=False, dir_path=self.WD, vocab=self.contribution.vocab, contribution=self.contribution) if self.data_model_num == 2: res, error_message, errors = ipmag.upload_magic2(dir_path=self.WD, data_model=self.er_magic.data_model) del wait if res: text = "You are ready to upload!\n{} was generated in {}".format(os.path.split(res)[1], os.path.split(res)[0]) dlg = pw.ChooseOne(self, "Go to MagIC for uploading", "Not ready yet", text, "Saved") del wait #dlg = wx.MessageDialog(self, caption="Saved", message=text, style=wx.OK) else: text = "There were some problems with the creation of your upload file.\nError message: {}\nSee Terminal/message window for details".format(error_message) dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) dlg.Centre() result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy() if result == wx.ID_YES: pw.on_database_upload(None) if self.data_model_num == 3: if not res: from programs import magic_gui self.Disable() self.Hide() self.magic_gui_frame = magic_gui.MainFrame(self.WD, dmodel=self.data_model, title="Validations", contribution=self.contribution) self.magic_gui_frame.validation_mode = ['specimens'] self.magic_gui_frame.failing_items = all_failing_items self.magic_gui_frame.change_dir_button.Disable() self.magic_gui_frame.Centre() self.magic_gui_frame.Show() self.magic_gui_frame.highlight_problems(has_problems) # # change name of upload button to 'exit validation mode' self.magic_gui_frame.bSizer2.GetStaticBox().SetLabel('return to main GUI') self.magic_gui_frame.btn_upload.SetLabel("exit validation mode") # bind that button to quitting magic gui and re-enabling Pmag GUI self.magic_gui_frame.Bind(wx.EVT_BUTTON, self.on_end_validation, self.magic_gui_frame.btn_upload) # do binding so that closing/quitting re-opens the main frame self.magic_gui_frame.Bind(wx.EVT_CLOSE, self.on_end_validation) # this makes it work with only the validation window open self.magic_gui_frame.Bind(wx.EVT_MENU, lambda event: self.menubar.on_quit(event, self.magic_gui_frame), self.magic_gui_frame.menubar.file_quit) # this makes it work if an additional grid is open self.Bind(wx.EVT_MENU, lambda event: self.menubar.on_quit(event, self.magic_gui_frame), self.magic_gui_frame.menubar.file_quit)
def on_upload_file(self, event): """ Write all data to appropriate er_* and pmag_* files. Then use those files to create a MagIC upload format file. Validate the upload file. """ # coherence validations wait = wx.BusyInfo('Validating data, please wait...') wx.SafeYield() spec_warnings, samp_warnings, site_warnings, loc_warnings = self.er_magic.validate_data() result_warnings = self.er_magic.validate_results(self.er_magic.results) meas_warnings = self.er_magic.validate_measurements(self.er_magic.measurements) self.warn_dict = {'specimen': spec_warnings, 'sample': samp_warnings, 'site': site_warnings, 'location': loc_warnings, 'result': result_warnings, 'age': {}, 'measurement': meas_warnings} # done coherence validations del wait # write upload file and perform data validations wait = wx.BusyInfo('Making upload file, please wait...') wx.SafeYield() self.er_magic.write_files() upfile, error_message, errors = ipmag.upload_magic(dir_path=self.WD, data_model=self.data_model) del wait if upfile: text = "You are ready to upload.\nYour file:\n{}\nwas generated in directory: \n{}\nDrag and drop this file in the MagIC database.".format(os.path.split(upfile)[1], self.WD) dlg = wx.MessageDialog(self, caption="Saved", message=text, style=wx.OK) else: text = "There were some problems with the creation of your upload file.\nError message: {}\nSee Terminal/Command Prompt for details".format(error_message) dlg = wx.MessageDialog(self, caption="Error", message=text, style=wx.OK) result = dlg.ShowModal() if result == wx.ID_OK: dlg.Destroy() self.edited = False ## add together data & coherence errors into one dictionary if errors: for item_type in errors: for item_name in errors[item_type]: if item_name in self.warn_dict[item_type]: self.warn_dict[item_type][item_name].update(errors[item_type][item_name]) else: self.warn_dict[item_type][item_name] = errors[item_type][item_name] has_problems = [] for item_type, warnings in list(self.warn_dict.items()): if warnings: has_problems.append(item_type) # for any dtypes with validation problems (data or coherence), # highlight the button to the corresponding grid # skip this step for Windows if sys.platform in ['win32', 'win62']: pass else: for dtype in self.warn_dict: wind = self.FindWindowByName(dtype + '_btn') if wind: if dtype in has_problems: wind.Bind(wx.EVT_PAINT, self.highlight_button) else: wind.Unbind(wx.EVT_PAINT, handler=self.highlight_button) self.Refresh() if has_problems: self.validation_mode = set(has_problems) if sys.platform in ['win32', 'win62']: self.message.SetLabel('The following grid(s) have incorrect or incomplete data:\n{}'.format(', '.join(self.validation_mode))) else: self.message.SetLabel('Highlighted grids have incorrect or incomplete data') self.bSizer_msg.ShowItems(True) self.hbox.Fit(self) if not has_problems: self.validation_mode = set() self.message.SetLabel('') self.bSizer_msg.ShowItems(False) self.hbox.Fit(self)
def test_with_different_input_output_dir(self): input_dir_path = os.path.join(WD, 'data_files', '3_0', 'McMurdo') outfile, error_message, errors, all_errors = ipmag.upload_magic(dir_path=".", input_dir_path=input_dir_path) msg = "Validation of your upload file has failed.\nYou can still upload" self.assertTrue(error_message.startswith(msg)) self.assertTrue(glob.glob("McMurdo*.txt"))
def test3_with_invalid_files(self): dir_path = os.path.join(WD, 'data_files', '3_0', 'Megiddo') outfile, error_message, errors, all_errors = ipmag.upload_magic(dir_path=dir_path) msg = "Validation of your upload file has failed.\nYou can still upload" self.assertTrue(error_message.startswith(msg))