def move_to_bad_logs(filename): """moves filename into BAD_LOGS dir""" bad_logs_dir = os.path.join(os.path.dirname(filename), '..', 'bad_logs') bad_logs_dir = os.path.abspath(bad_logs_dir) mkdir_p(bad_logs_dir) name = os.path.split(filename)[1] dst_file = os.path.join(bad_logs_dir, name) log.debug("%s => %s", filename, dst_file) os.rename(filename, dst_file)
def write_to_json(events_dir, data): """writes data to a json file; the file name is: <EVENTS_DIR>/event/instance, event and instance are provided by data itself""" event = data['eventName'] instance = data['instances'] filename = os.path.join(events_dir, event, instance) mkdir_p(os.path.dirname(filename)) if not os.path.exists(filename): with open(filename, 'w') as f_out: json.dump(data, f_out) elif data['eventTime'] > get_time_from_file(filename): # replace old event with current one with open(filename, 'w') as f_out: json.dump(data, f_out)
def write_to_disk(cache_dir, key): """write key to disk in cache_dir""" dst = os.path.join(cache_dir, key.name) mkdir_p(os.path.dirname(dst)) # key.get_contents_to_filename() is a blocking function, # if we try to download non existing files, it will hang here # it works only on unix systems signal.signal(signal.SIGALRM, _timeout) if not os.path.exists(dst): log.debug('downloading: {0}'.format(key.name)) signal.alarm(GET_CONTENTS_TO_FILENAME_TIMEOUT) try: key.get_contents_to_filename(dst) except TimeoutException: log.debug('timeout downloading: {0}'.format(key.name)) else: # file is already cached locally log.debug('{0} is already cached'.format(key.name))