def run(action, dburl, eventws, minmag, minlat, maxlat, minlon, maxlon, start, end, stimespan, search_radius, channels, min_sample_rate, s_inventory, traveltime_phases, wtimespan, processing, advanced_settings, isterminal=False): """ Main run method. KEEP the ARGUMENT THE SAME AS THE config.yaml OTHERWISE YOU'LL GET A DIFFERENT CONFIG SAVED IN THE DB :param processing: a dict as load from the config """ _args_ = dict(locals()) # this must be the first statement, so that we catch all arguments and # no local variable (none has been declared yet). Note: dict(locals()) avoids problems with # variables created inside loops, when iterating over _args_ (see below) if action == 'gui': from stream2segment.gui import main as main_gui main_gui.run_in_browser(dburl) return 0 session = get_session(dburl, scoped=True) # FIXME: is it necessary for multiprocessing in processing? # create logger handler run_row = config_logger_and_return_run_instance(session, isterminal) yaml_dict = load_def_cfg() # update with our current variables (only those present in the config_yaml): yaml_dict.update(**{k: v for k, v in _args_.iteritems() if k in yaml_dict}) # print local vars: yaml_content = StringIO() # use safe_dump to avoid python types. See: # http://stackoverflow.com/questions/1950306/pyyaml-dumping-without-tags yaml_content.write(yaml.safe_dump(yaml_dict, default_flow_style=False)) config_text = yaml_content.getvalue() if isterminal: print("Arguments:") tab = " " print(tab + config_text.replace("\n", "\n%s" % tab)) run_row.config = tounicode(config_text) session.commit() # udpate run row. flush might be also used but we prever sotring to db ret = 0 try: if s_inventory is None: sta_del = session.query(models.Station).\ filter(models.Station.inventory_xml != None).\ update({models.Station.inventory_xml.key: None}) if sta_del: session.commit() logger.info("Deleted %d station inventories (set to null)" % sta_del) segments = [] if 'd' in action: starttime = time.time() ret = query_main(session, run_row.id, eventws, minmag, minlat, maxlat, minlon, maxlon, start, end, stimespan, search_radius['minmag'], search_radius['maxmag'], search_radius['minradius'], search_radius['maxradius'], channels, min_sample_rate, s_inventory is True, traveltime_phases, wtimespan, advanced_settings, isterminal) logger.info("Download completed in %s", tdstr(dt.timedelta(seconds=time.time()-starttime))) if 'p' in action.lower() and ret == 0: starttime = time.time() if 'P' in action: try: _ = session.query(models.Processing).delete() # returns num rows deleted session.commit() except SQLAlchemyError: session.rollback() raise Exception("Unable to delete all processing (internal db error). Please" "try to run again the program") segments = session.query(models.Segment).\ filter(~models.Segment.processings.any()).all() # @UndefinedVariable process_main(session, segments, run_row.id, isterminal, **processing) logger.info("Processing completed in %s", tdstr(dt.timedelta(seconds=time.time()-starttime))) logger.info("") logger.info("%d total error(s), %d total warning(s)", run_row.errors, run_row.warnings) except Exception as exc: logger.critical(str(exc)) raise finally: for handler in logger.handlers: try: handler.close() except (AttributeError, TypeError, IOError, ValueError): pass return 0
def tdstr(timdelta): """Returns a formatted timedelta with seconds rounded up or down""" # remainder. timedelta has already a nicer formatting with its str method: # str(timedelta(hours=15000,seconds=4500)) # >>> '625 days, 1:15:00' # str(timedelta(seconds=4500) - timedelta(microseconds=1)) # >>> '1:14:59.999999' # so we just need to append 'hours' and round microseconds add = 1 if timdelta.microseconds >= 500000 else 0 str_ = str(dt.timedelta(days=timdelta.days, seconds=timdelta.seconds+add, microseconds=0)) spl = str_.split(":") return str_ if len(spl) != 3 else "%sh:%sm:%ss" % (spl[0], spl[1], spl[2]) # a bit hacky maybe, should be checked: cfg_dict = load_def_cfg() # IMPORTANT # IMPORTANT: THE ARGUMENT NAMES HERE MUST BE THE SAME AS THE CONFIG FILE!!! # IMPORTANT def run(action, dburl, eventws, minmag, minlat, maxlat, minlon, maxlon, start, end, stimespan, search_radius, channels, min_sample_rate, s_inventory, traveltime_phases, wtimespan, processing, advanced_settings, isterminal=False): """ Main run method. KEEP the ARGUMENT THE SAME AS THE config.yaml OTHERWISE YOU'LL GET A DIFFERENT CONFIG SAVED IN THE DB :param processing: a dict as load from the config """ _args_ = dict(locals()) # this must be the first statement, so that we catch all arguments and