Exemplo n.º 1
0
def main_csv_reader(args):
    path_to_coffee = args.path_to_coffee
    path_to_matched = args.matched_json
    all_people_list = flat_list(list(read_csv_file(path_to_coffee)))
    matched_in_this_session = []
    error = False

    if path_to_matched:
        try:
            matched_people_json = read_json_file(path_to_matched)
            tuple_list = create_tuple_list(all_people_list,
                                           matched_people_json)
            sorted_people_list = sort_tuple_list(tuple_list)
        except:
            raise ('Only use the program generated matched_people.json file')
    else:
        write_json_file()
        matched_people_json = read_json_file('matched_people.json')
        sorted_people_list = all_people_list

    unmatched_people = []

    for person in sorted_people_list:
        if person not in matched_in_this_session:
            individual_match_list = invidual_preproc(person, all_people_list,
                                                     matched_people_json,
                                                     matched_in_this_session)
            if individual_match_list:
                matched_pair = coffee_roulette(person, individual_match_list)
                if matched_pair is not None:
                    for person in matched_pair:
                        matched_in_this_session.append(person)
                else:
                    error = True
                    break
            else:
                unmatched_people.append(person)
        else:
            pass

    if error is False:
        create_today_matched(matched_in_this_session)
        if unmatched_people:
            create_today_unmatched(unmatched_people)

        updated_json = update_current_json(matched_people_json,
                                           matched_in_this_session)
        summary = "\n{} Matches".format(date.today())
        summary = create_matched_people_string(matched_in_this_session,
                                               summary)
        summary_messsage, alone = make_summary(matched_in_this_session,
                                               unmatched_people, summary, "")
        summary += alone
        write_json_file(updated_json)
        write_txt_file(summary)
        print(summary_messsage)
Exemplo n.º 2
0
    def check_diff_projects(self):
        """
            Load every current and previous attribute reports and
            creates a report if any diff is found.
        """
        results = []

        for report_path in self.report_list:

            report_file = util.get_basename_file(report_path)
            self.report_names.add(report_file)

            resource, attribute = util.extract_resource_info(report_file)

            data = util.read_json_file(report_path)
            previous_data, previous_report_file = self._retrieve_reports_to_be_compared (report_file)

            if previous_data:
                self.previous_report_names.add(previous_report_file)
                diff = util.jsonfy(util.get_diff_dicts(data, previous_data))

                if diff:
                    util.print_to_stdout("Found diff for {0} in {1}.".format(attribute, resource), color="green")
                    results.append(self._generate_diff_projects_report(resource, attribute, diff, data))

        return results
Exemplo n.º 3
0
def main():
    options = get_arguments()

    print('\n        --:[ TERRORTIME PRIVATE KEY RECOVER ]:--')
    print('                          0x8861\n')

    keyparams = Keyparams(default_keyparams)
    if options.keyparams:
        keyparams = Keyparams(util.read_json_file(options.keyparams))

    keyparams = init(keyparams)

    with open(options.pubkey_filename, 'r') as f:
        pem_pubkeys = f.read().split(':')

    print('[+] %d public keys found in the file %s\n' %
          (len(pem_pubkeys), options.pubkey_filename))

    count = 0
    for i, pem_pubkey in enumerate(pem_pubkeys):
        output_filename = '{0}_{1:02d}'.format(options.output_filename, i + 1)
        is_success = export_keys(pem_pubkey, keyparams, output_filename)
        if is_success:
            count += 1
        print('')

    print('[+] %d private key(s) recovered' % count)
Exemplo n.º 4
0
 def __init_json_data(self):
     if not os.path.exists(self.__node_pos_path):
         util.write_json_file(file_path=self.__node_pos_path, data={})
         logging.info("创建文件 - %s" % self.__node_pos_path)
     else:
         self.node_pos_data = util.read_json_file(
             file_path=self.__node_pos_path)
Exemplo n.º 5
0
	def get_robots_data(self):
		try:
			self.robots_data = read_json_file(file_path=config.robots_json_data_path)
			self.has_robot_data = True
			return True
		except FileNotFoundError:
			print("File Not Found,Must Enter Info By Yourself")
			return False
Exemplo n.º 6
0
 def __init__(self, message_service: MessageGenerator,
              plot_service: Plotter, update_service: UpdateService):
     self.message_service: MessageGenerator = message_service
     self.plot_service: Plotter = plot_service
     self.update_service: UpdateService = update_service
     self.updater: Updater = None
     # Whitelist jener leute, die die DSGVO akzeptiert haben
     self.whitelist = util.read_json_file("whitelist.json")["list"]
Exemplo n.º 7
0
 def get_cookies(self, platform):
     try:
         cookies = util.read_json_file(
             file_path=config.cookies_path)[platform]
         return cookies
     except KeyError or FileNotFoundError:
         self.log_text_send.emit("get cookies error")
         print("get cookies error")
         return
Exemplo n.º 8
0
    def __init_robots(self):
        value_count = 0
        if self.__start_uid > 0:
            uid_list = list()
            for uid in [
                    uid for uid in range(self.__start_uid, self.__start_uid +
                                         self.__robots_num)
            ]:
                player = Robot(sid=self.__sid, uid=uid, ksid=self.__sid)
                if player.has_login:
                    self.robot_object_list.append(player)
                    uid_list.append(uid)
                    value_count += 1
                else:
                    logging.error("%s login failed" % uid)
        else:
            uid_list = []
            robot_count = 0
            while robot_count < self.__robots_num:
                player = Robot(sid=self.__sid, uid=0, ksid=self.__sid)
                if player.has_login:
                    self.robot_object_list.append(player)
                    uid_list.append(player.uid)
                    value_count += 1
                else:
                    logging.error("create account failed")
                robot_count += 1
        logging.info("login success num: %s ; total num : %s" %
                     (value_count, self.__robots_num))
        try:
            self.__robots_data = read_json_file(
                file_path=config.robots_json_data_path)
        except FileNotFoundError:
            self.__robots_data[str(config.okc_environment)] = {
                str(self.__sid): uid_list
            }
        logging.info("robots_data : %s" % self.__robots_data)
        try:
            old_uid = set(self.__robots_data[str(config.okc_environment)][str(
                self.__sid)])
            new_uid = set(uid_list)
            self.__robots_data[str(config.okc_environment)][str(
                self.__sid)] = list(old_uid | new_uid)
        except KeyError:
            import traceback
            logging.error(traceback.print_exc())

        last_load = {
            "env": config.okc_environment,
            "kingdom": self.__sid,
            "sid": self.__sid,
            "uid": uid_list
        }
        self.__robots_data["last_load"] = last_load
        write_json_file(file_path=config.robots_json_data_path,
                        data=self.__robots_data)
Exemplo n.º 9
0
 def __save_robots(self):
     env_path = util.get_ini_data(ini_path=config.conf_path,
                                  section="path",
                                  section_item="okc_environment")
     last_load = util.read_json_file(env_path)
     if self.robots:
         robot_uid_list = []
         for robot in self.robots:
             robot_uid_list.append(robot.uid)
         last_load["uid"] = robot_uid_list
         # data = {"sid": self.sid, "player_list": robot_uid_list}
         util.write_json_file(file_path=env_path, data=last_load)
    def predict(self, image_path, top_k, cat_to_name_path):
        ''' Predict the class (or classes) of an image using a trained deep learning model.
        '''
        self.model.to(self.device)

        im = Image.open(image_path)
        im_tensor = util.process_image(im)
    #     print(im_tensor.shape)

        im_tensor.unsqueeze_(0)
#         print(im_tensor.shape)

#         log_ps = self.model.forward(im_tensor.float())
        im_tensor = im_tensor.to(self.device)
    
        # get probabilities without gradients
        with torch.no_grad():
            log_ps = self.model.forward(im_tensor.float())
        ps = torch.exp(log_ps)

        # get top probabilities
        top_probs, top_indexes = ps.topk(top_k)
    #     print(f'top_probs: {top_probs}') 
    #     print(f'top_indexes: {top_indexes}')      

        # flatten
        top_probs = top_probs.cpu().numpy().reshape(top_k,)
    #     print(f'top_probs: {top_probs}')

        top_indexes = top_indexes.cpu().numpy().reshape(top_k,)
    #     print(f'top_indexes: {top_indexes}')

        #print(f'model.class_to_idx: {model.class_to_idx}')

        idx_to_class = {v: str(k) for k, v in self.class_to_idx.items()}
        #print(f'idx_to_class: {idx_to_class}')

        top_classes = []
        for tc in top_indexes: top_classes.append(idx_to_class[tc])
            
        #[idx_to_class[tc] for tc in top_indexes.cpu().numpy()]
        
        # get class names
        class_to_name = util.read_json_file(cat_to_name_path)
        
        return top_probs, [class_to_name[tc] for tc in top_classes]
   
        
        
        
    
        
Exemplo n.º 11
0
	def mail_send(self, player_name):
		
		mail_type = 0
		
		import util
		import random
		chat_path = util.get_ini_data(conf_path, section="path", section_item="chat_path")
		chat_data = util.read_json_file(file_path=chat_path)
		
		data_key = str(random.randint(0, len(chat_data) - 1))
		content = chat_data[data_key]
		
		return self.__request.mail_send(mail_type=mail_type, target_name=player_name, content=content)
Exemplo n.º 12
0
 def setUpClass(self, mock_sm):
     os.environ['KB_WORKSPACE_ID'] = '12345'  # That's the same workspace as my luggage!
     specs_list = read_json_file('data/specs.json')
     specs_dict = dict()
     for s in specs_list:
         specs_dict[s['info']['id']] = s
     mock_sm.return_value.app_specs = {'release': specs_dict, 'beta': specs_dict, 'dev': specs_dict}
     # mock_njs.return_value.list_methods_spec.return_value = read_narrative_file('data/specs.json')
     self.wm = WidgetManager()
     self.good_widget = "kbaseTabTable"
     self.bad_widget = "notAWidget"
     self.good_tag = "release"
     self.bad_tag = "notATag"
Exemplo n.º 13
0
    def _retrieve_reports_to_be_compared(self, report_file):
        """
            Loop over the previous reports to find the current report
            and retrieve and return this data.
        """
        for previous_report_path in self.previous_report_list:

                previous_report_file = util.get_basename_file(previous_report_path)

                if report_file == previous_report_file:
                    previous_data = util.read_json_file(previous_report_path)

                    return previous_data, previous_report_file

        return None, None
Exemplo n.º 14
0
    def _get_resources_list(self, resource_name):
        """
            Search for a all the reports for a given resource,
            loading and returning them in a list.
        """
        resources_list = []

        for report_path in self.report_list:

            project, resource = util.extract_resource_info(util.get_basename_file(report_path))

            if resource_name == resource:
                resources_list.append([project, util.read_json_file(report_path)])

        return resources_list
Exemplo n.º 15
0
 def setUpClass(self, mock_sm):
     os.environ[
         'KB_WORKSPACE_ID'] = '12345'  # That's the same workspace as my luggage!
     specs_list = read_json_file('data/specs.json')
     specs_dict = dict()
     for s in specs_list:
         specs_dict[s['info']['id']] = s
     mock_sm.return_value.app_specs = {
         'release': specs_dict,
         'beta': specs_dict,
         'dev': specs_dict
     }
     # mock_njs.return_value.list_methods_spec.return_value = read_narrative_file('data/specs.json')
     self.wm = WidgetManager()
     self.good_widget = "kbaseTabTable"
     self.bad_widget = "notAWidget"
     self.good_tag = "release"
     self.bad_tag = "notATag"
Exemplo n.º 16
0
    def __init__(self):
        self.app_name = config.okc_app_name

        self.default_wait_time = 1.0
        self.default_save_path = "G:/TestData/okc_test/new_teach/"
        self.ui_pos_path = util.get_ini_data(ini_path=config.conf_path,
                                             section="path",
                                             section_item="ui_pos")
        # util.get_ini_data(config.conf_path, section="path", section_item="okc_test_image_dir")

        self.phone = PhoneManager()
        self.ui_node = UiNodeData()

        self.android_object = None
        self.poco: UnityPoco = None

        self.click_wait_time = 0.3
        self.target_node_pos = tuple()

        self.ui_pos = util.read_json_file(file_path=self.ui_pos_path)
        self.start_game()
Exemplo n.º 17
0
    def smart_run(self, strategy="combined"):
        """
        running window merging

        :param strategy: strategy to merge windows. values could be 
            "combined" and "selective". 
            If "combined", each channel will be kept and the weighting will be
            calculated according to the number of windows. 
            If "selective", only one channel(the one with max number of window)
            will be kept and the weighting will be 1. If there are multiple max
            values, the channel_id with smaller location value will be kept.
        :type strategy: string
        """
        strategy = strategy.lower()
        if strategy not in ['combined', 'selective']:
            raise ValueError("strategy can only be: 1)'combined'; "
                             "2)'selective'")

        dirlist = read_json_file(self.dirfile)
        if isinstance(dirlist, list):
            for _dir in dirlist:
                self.merge_window(_dir, strategy=strategy)
        elif isinstance(dirlist, JSONObject):
            self.merge_window(dirlist, strategy)
Exemplo n.º 18
0
    def smart_run(self, strategy="combined"):
        """
        running window merging

        :param strategy: strategy to merge windows. values could be 
            "combined" and "selective". 
            If "combined", each channel will be kept and the weighting will be
            calculated according to the number of windows. 
            If "selective", only one channel(the one with max number of window)
            will be kept and the weighting will be 1. If there are multiple max
            values, the channel_id with smaller location value will be kept.
        :type strategy: string
        """
        strategy = strategy.lower()
        if strategy not in ['combined', 'selective']:
            raise ValueError("strategy can only be: 1)'combined'; "
                             "2)'selective'")

        dirlist = read_json_file(self.dirfile)
        if isinstance(dirlist, list):
            for _dir in dirlist:
                self.merge_window(_dir, strategy=strategy)
        elif isinstance(dirlist, JSONObject):
            self.merge_window(dirlist, strategy)
Exemplo n.º 19
0
    def __init_robots(self):
        if self.__start_uid != 0 and self.__robots_num > 0:
            count = 0
            value_count = 0
            while count < self.__robots_num:
                robot_obj: Robot = Robot(self.sid, self.__start_uid)
                if self.__start_uid != -1:
                    self.__start_uid += 1
                count += 1
                if robot_obj.has_login:
                    self.robots.append(robot_obj)
                    value_count += 1
            logging.info("login success num: %s ; total num : %s" %
                         (value_count, self.__robots_num))
            print("-" * 150)

        elif os.path.exists(self.__robots_path):

            user_info = util.read_json_file(self.__robots_path)
            try:
                sid = int(user_info["sid"])
                uid_list = list(user_info["player_list"])
                value_count = 0
                for uid in uid_list:
                    robot_obj: Robot = Robot(sid, uid)
                    if robot_obj.has_login:
                        self.robots.append(robot_obj)
                        value_count += 1
                logging.info("login success num: %s ; total num : %s" %
                             (value_count, len(uid_list)))
                print("-" * 150)

            except KeyError:
                return
        else:
            return
Exemplo n.º 20
0
def walkt_tree(toplevel, directory, pattern, after):
    nf = 0
    for p in sorted(directory.rglob(pattern)):
        s = p.stem
        if s.endswith(".geojson"):
            s = s.rsplit(".", 1)[0]
        stid, day, tim = s.split("_")
        ts = ciso8601.parse_datetime(day + " " + tim + "-00:00").timestamp()

        if ts < after:
            # print("skipping", s, file=sys.stderr)
            continue

        # print(stid, day, tim, datetime.fromtimestamp(ts, pytz.utc))
        if toplevel.endswith("madis/"):
            typus = "netCDF"
        if toplevel.endswith("gisc/"):
            typus = "BUFR"
        entry = {"source": typus, "syn_timestamp": int(ts)}
        gj = None
        if stid not in station_list:
            # maybe mobile. Check ascent for type
            # example unregistered, but obviously fixed:
            # https://radiosonde.mah.priv.at/data-dev/gisc/08/383/08383_20210206_120000.geojson
            # check id syntax - 5 digits = unregistered else mobile
            if re.match(r"^\d{5}$", stid):
                # WMO id syntax, but not in station_list
                # hence an unregistered but fixed station
                idtype = "unregistered"
            else:
                # could be ship registration syntax. Check detail file.
                gj = util.read_json_file(p, asGeojson=True, useBrotli=True)
                idtype = gj.properties["id_type"]
                # propagate per-ascent coords down to ascent
                entry["lat"] = round(gj.properties["lat"], 6)
                entry["lon"] = round(gj.properties["lon"], 6)
                entry["elevation"] = round(gj.properties["elevation"], 2)
        else:
            # registered
            st = station_list[stid]
            idtype = "wmo"

        if stid not in flights:
            flights[stid] = Feature(
                # FIXME add point after sorting for mobiles
                properties={"ascents": [entry]})
        else:
            flights[stid].properties["ascents"].append(entry)

        f = flights[stid]
        f.properties["station_id"] = stid
        f.properties["id_type"] = idtype
        f.properties["name"] = stid

        if stid in station_list:
            st = station_list[stid]
            # override name if we have one
            f.properties["name"] = st["name"]
        else:
            if idtype == "unregistered":
                if stid not in missing:
                    locations = rg.search((st["lat"], st["lon"]),
                                          verbose=False)
                    if locations:
                        print(
                            stid,
                            locations,
                            st["lat"],
                            st["lon"],
                            st["elevation"],
                            file=sys.stderr,
                        )
                        loc = locations[0]
                        f.properties["name"] = loc["name"] + ", " + loc["cc"]
                        s = f'{stid.rjust(11, "X")} {st["lat"]} {st["lon"]} {st["elevation"]} {f.properties["name"]} 2020'
                        txtfrag.append(s)
                    missing[stid] = {
                        "name": f.properties["name"],
                        "lat": st["lat"],
                        "lon": st["lon"],
                        "elevation": st["elevation"],
                    }
        # this needs fixing up for mobiles after sorting
        f.geometry = Point((round(st["lon"],
                                  6), round(st["lat"],
                                            6), round(st["elevation"], 1)))
        nf += 1
    return (nf, 1, 1)
Exemplo n.º 21
0
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import config
import util
import logging

from okc_robot.data import number_class as Num
from okc_robot.data.number_class import okc_number_dict

# read game.json
game_json_data_path = util.get_ini_data(ini_path=config.conf_path, section="path", section_item="game_json")
game_json_data = util.read_json_file(file_path=game_json_data_path)

# read error.json
error_json_data_path = util.get_ini_data(ini_path=config.conf_path, section="path", section_item="error_json")
error_json_data = util.read_json_file(file_path=error_json_data_path)

# read wild_res.json
wild_res_json_data_path = util.get_ini_data(config.conf_path, section="path", section_item="wild_res")
wild_res_json_data = util.read_json_file(file_path=wild_res_json_data_path)

# read equip.json
equip_json_data_path = util.get_ini_data(ini_path=config.conf_path, section="path", section_item="equip_json")
equip_json_data = util.read_json_file(file_path=equip_json_data_path)

# 动态创建GameJson类, game_json_data is dict
GameJson = type("GameJson", (object,), game_json_data)


class GameNumerical(GameJson):
	
Exemplo n.º 22
0
request_interval = 0.1

okc_environment = OkcEnvironment.Dev.value

okc_request_mode = OkcRequestMode.Http.value

robots_json_data_path = util.get_ini_data(ini_path=conf_path,
                                          section="path",
                                          section_item="robots_json")

# read game.json
game_json_data_path = util.get_ini_data(ini_path=conf_path,
                                        section="path",
                                        section_item="game_json")
game_json_data = util.read_json_file(file_path=game_json_data_path)

# read error.json
error_json_data_path = util.get_ini_data(ini_path=conf_path,
                                         section="path",
                                         section_item="error_json")
error_json_data = util.read_json_file(file_path=error_json_data_path)

# read wild_res.json
wild_res_json_data_path = util.get_ini_data(conf_path,
                                            section="path",
                                            section_item="wild_res")
wild_res_json_data = util.read_json_file(file_path=wild_res_json_data_path)

# read equip.json
equip_json_data_path = util.get_ini_data(ini_path=conf_path,
Exemplo n.º 23
0
def okc_robot_start():
    print(
        "welcome to use okc robot.\n\nthe environment : 0-test ,1-dev ,2-online test ,3-online ."
        .title())
    print(
        "no input or input error Use the environment where you last logged in."
        .title())
    print("If this is your first login, use test environment.\n".title())

    input_env = input("choose environment :\n".title())
    sid = input("enter sid:\n".title())
    start_uid = input("start uid:\n".title())
    robot_num = input("robot num:\n".title())

    env_path = util.get_ini_data(ini_path=config.conf_path,
                                 section="path",
                                 section_item="okc_environment")

    def get_help():
        help_info = "-" * 150 + "\n"
        for command_name in robots.__class__.__dict__.keys():
            if not re.match(r"__", command_name):
                help_info += "Command : %s  Params : %s" % (
                    command_name + " " * (30 - len(command_name)) + "·" * 10,
                    robots.__getattribute__(command_name).__doc__)
                help_info += "\n" + "-" * 150 + "\n"
        return help_info

    try:
        input_env = int(input_env)
        if input_env == 0:
            config.okc_environment = config.OkcEnvironment.Test.value
        elif input_env == 1:
            config.okc_environment = config.OkcEnvironment.Dev.value
        elif input_env == 2:
            config.okc_environment = config.OkcEnvironment.OnlineTest.value
        elif input_env == 3:
            config.okc_environment = config.OkcEnvironment.Online.value
        else:
            config.okc_environment = config.OkcEnvironment.Test.value
        logging.info("environment : %s " % config.ENV[config.okc_environment])
        sid = int(sid)
        start_uid = int(start_uid)
        player_num = int(robot_num)
        data = {
            "env": config.okc_environment,
            "sid": sid,
            "uid": list(range(start_uid, start_uid + player_num))
        }
        util.write_json_file(env_path, data=data)
        robots = AnalysisCommand(sid=sid,
                                 start_uid=start_uid,
                                 robot_num=player_num)
    except ValueError:
        last_load = util.read_json_file(env_path)

        env = int(last_load["env"])
        config.okc_environment = env

        logging.info("environment : %s " % config.ENV[config.okc_environment])
        print("-" * 150)

        sid = int(last_load["sid"])
        start_uid = last_load["uid"][0]
        player_num = len(last_load["uid"])

        if start_uid == -1:
            robots = AnalysisCommand()
        else:
            robots = AnalysisCommand(sid=sid,
                                     start_uid=start_uid,
                                     robot_num=player_num)

    if not robots.robots:
        print("Robot Init Failed")
        return
    while True:
        command = input("enter command : \n".title())
        cmd = re.match(r"(\w+)(:?\s+(.*))", command)
        if not cmd:
            print("Input Error.\n")
            print(get_help())
            continue
        try:
            cmd_name, params = cmd.group(1, 2)
        except ValueError:
            print("Input Params Error.\n")
            print(get_help())
            continue
        cmd_list = robots.__dir__()
        if cmd_name in cmd_list:
            cmd_value = getattr(robots, cmd_name)
            try:
                cmd_value(params)
            except:
                logging.error(traceback.print_exc())
        else:
            print("command -> '%s' isn't in cmd list.\n" % cmd_name)
            print(get_help())
            continue
Exemplo n.º 24
0
def get_leture_resoucre(course):
    path = os.path.join(data_path, course)
    path += ".json"
    if os.path.exists(path):
        return util.read_json_file(path)
Exemplo n.º 25
0
import util

api_key = util.read_json_file("api-key.json")
strings = util.read_json_file("strings.json")  # Strings einlesen
conf = util.read_json_file("config.json")  # config einlesen
command_file = util.read_json_file("commands.json")
consts = util.read_json_file("constants.json")

commands = command_file["command_list"]
category_list = command_file["categories"]
Exemplo n.º 26
0
 def __init__(self, data_handler: DataHandler,
              message_generator: MessageGenerator):
     self.data_handler = data_handler
     self.message_generator = message_generator
     self.subscriptions = util.read_json_file("subscribers.json")
     self.last_update = util.read_json_file("last-update.json")
Exemplo n.º 27
0
def main():
    parser = argparse.ArgumentParser(
        description="decode radiosonde BUFR and netCDF reports", add_help=True
    )
    parser.add_argument("-v", "--verbose", action="store_true", default=False)
    parser.add_argument(
        "--hstep",
        action="store",
        type=int,
        default=100,
        help="generate output only if samples vary vertically more than hstep",
    )
    parser.add_argument("--destdir", action="store", default=".")
    parser.add_argument(
        "--station",
        action="store",
        default=None,
        help="extract a single station by WMO id",
    )
    parser.add_argument("--geojson", action="store_true", default=False)
    parser.add_argument("--dump-geojson", action="store_true", default=False)
    parser.add_argument(
        "--sim-housekeep",
        action="store_true",
        default=False,
        help="just list what would happen to spooldirs; no input file processing",
    )
    parser.add_argument("--only-args", action="store_true", default=False)
    parser.add_argument("--summary", action="store", required=True)
    parser.add_argument(
        "-n",
        "--ignore-timestamps",
        action="store_true",
        help="ignore, and do not create timestamps",
    )
    parser.add_argument(
        "--stations",
        action="store",
        required=True,
        help="path to station_list.json file",
    )
    parser.add_argument("--tmpdir", action="store", default=None)
    parser.add_argument(
        "--max-age",
        action="store",
        type=int,
        default=config.MAX_DAYS_IN_SUMMARY,
        help="number of days of history to keep in summary",
    )
    parser.add_argument(
        "--keep-time",
        action="store",
        type=int,
        default=config.KEEP_MADIS_PROCESSED_FILES,
        help="time in secs to retain processed .gz files in MADIS incoming spooldir",
    )
    parser.add_argument("files", nargs="*")

    args = parser.parse_args()
    if args.tmpdir:
        config.tmpdir = args.tmpdir

    level = logging.WARNING
    if args.verbose:
        level = logging.DEBUG

    logging.basicConfig(level=level)
    os.umask(0o22)

    try:
        with pidfile.Pidfile(config.LOCKFILE, log=logging.debug, warn=logging.debug):

            station_dict = json.loads(util.read_file(args.stations).decode())
            updated_stations = []

            useBrotli = args.summary.endswith(".br")
            summary = util.read_json_file(
                args.summary, useBrotli=useBrotli, asGeojson=True
            )

            if args.only_args:
                flist = args.files
            else:
                l = list(
                    pathlib.Path(config.SPOOLDIR_GISC + config.INCOMING).glob("*.zip")
                )
                l.extend(
                    list(
                        pathlib.Path(config.SPOOLDIR_GISC_TOKYO + config.INCOMING).glob(
                            "*.bufr"
                        )
                    )
                )
                l.extend(
                    list(
                        pathlib.Path(config.SPOOLDIR_MADIS + config.INCOMING).glob(
                            "*.gz"
                        )
                    )
                )
                flist = [str(f) for f in l]

            # work the backlog
            if not args.sim_housekeep:
                process_files(args, flist, station_dict, updated_stations)

            if not args.sim_housekeep and updated_stations:
                logging.debug(f"creating GeoJSON summary: {args.summary}")
                update_geojson_summary(args, station_dict, updated_stations, summary)

            if not args.only_args:
                logging.debug("running housekeeping")
                keep_house(args)
            return 0

    except pidfile.ProcessRunningException:
        logging.warning(f"the pid file {config.LOCKFILE} is in use, exiting.")
        return -1
Exemplo n.º 28
0
import json
from api_client import parser, client, content
from util import pretty_print, read_json_file

parser.add_argument('folder_path', type=str, help="Path to folder to sync to")
parser.add_argument('data_file', type=str, help="File to read contents from")
args = parser.parse_args()

api_client = client.ApiClient(args.access_id, args.access_key, args.endpoint)
content_api = content.ContentManagementApi(api_client)
folder_id = content_api.find_id_by_path(args.folder_path)
content_item = None
content_item = read_json_file(args.data_file)
content_api.sync(folder_id, content_item)