Beispiel #1
0
Datei: cloud.py Projekt: EQ4/DRR
def register_stream_list(reindex=False):
    # Find the local streams and make sure they are all registered in the sqlite3 database.
    #
    # Get the existing streams as a set
    #
    # If we are asked to re-index (due to trying to fix a bug) then we ignore what we have
    # and just go ahead and do everything.
    #
    if reindex:
        all_registered = Set([])

    else:
        all_registered = Set(DB.all('streams', ['name']))

    # There should be a smarter way to do this ... you'd think. We should also
    # be more faithfully giving things extensions since it's not 100% mp3
    all_files = Set(glob('%s/*.mp3' % misc.DIR_STREAMS))

    diff = all_files.difference(all_registered)

    # This is a list of files we haven't scanned yet...
    if not diff: return True

    # This basically means we could still be writing
    # this file.
    #
    # We take the cascade time and then buffer it by a minute, just
    # to be sure.
    #
    # If the creation time is less then this then we don't register this
    # until later.
    cutoff = time.mktime(
        (datetime.now() -
         timedelta(minutes=1, seconds=misc.config['cascadetime'])).timetuple())

    for fname in diff:
        if len(fname) == 0 or os.path.getctime(fname) > cutoff:
            next

        info = stream_info(fname)
        if not info:
            continue

        DB.register_stream(info)

        if not misc.manager_is_running():
            logging.info("Manager is gone, shutting down")
            raise Exception()
Beispiel #2
0
Datei: cloud.py Projekt: EQ4/DRR
def register_stream_list(reindex=False):
  # Find the local streams and make sure they are all registered in the sqlite3 database. 
  #
  # Get the existing streams as a set
  #
  # If we are asked to re-index (due to trying to fix a bug) then we ignore what we have
  # and just go ahead and do everything.
  #
  if reindex:
    all_registered = Set([])

  else: 
    all_registered = Set(DB.all('streams', ['name']))

  # There should be a smarter way to do this ... you'd think. We should also
  # be more faithfully giving things extensions since it's not 100% mp3
  all_files = Set(glob('%s/*.mp3' % misc.DIR_STREAMS))
 
  diff = all_files.difference(all_registered)

  # This is a list of files we haven't scanned yet...
  if not diff: return True

  # This basically means we could still be writing
  # this file.
  #
  # We take the cascade time and then buffer it by a minute, just
  # to be sure.
  # 
  # If the creation time is less then this then we don't register this
  # until later.
  cutoff = time.mktime((datetime.now() - timedelta(minutes=1, seconds=misc.config['cascadetime'])).timetuple())

  for fname in diff:
    if len(fname) == 0 or os.path.getctime(fname) > cutoff:
      next

    info = stream_info(fname)
    if not info:
      continue

    DB.register_stream(info)

    if not misc.manager_is_running():
      logging.info("Manager is gone, shutting down")
      raise Exception()
Beispiel #3
0
Datei: cloud.py Projekt: EQ4/DRR
def prune_process(lockMap, reindex=False, force=False):
  # This is internal, call prune() directly. This is a normally blocking
  # process that is prepared by prune(), making it easily callable asynchronously 
  # If another prune is running then we just bail
  if not lockMap['prune'].acquire(False) and not force:
    logging.warn("Tried to run another prune whilst one is running. Aborting")
    return True

  # If we are the first process then we need to make sure that the webserver is up before
  # we do this to check to see if we are official
  time.sleep(2)

  pid = misc.change_proc_name("%s-cleanup" % misc.config['callsign'])

  # We want to run the am_i_official here since it could block on a DNS lookup
  misc.am_i_official()

  try:
    register_stream_list(reindex)

  except:
    lockMap['prune'].release()
    return None

  db = DB.connect()

  archive_duration = misc.config['archivedays'] * TS.ONE_DAY_SECOND
  cutoff = TS.unixtime('prune') - archive_duration

  # Remove all slices older than 4 hours.
  slice_cutoff = TS.unixtime('prune') - 0.1667 * TS.ONE_DAY_SECOND

  cloud_cutoff = None
  if misc.config['cloud']:
    cloud_cutoff = TS.unixtime('prune') - misc.config['cloudarchive'] * TS.ONE_DAY_SECOND

  # Put thingies into the cloud.
  count = 0
  for file_name in glob('*/*.mp3'):
    #
    # Depending on many factors this could be running for hours
    # or even days.  We want to make sure this isn't a blarrrghhh
    # zombie process or worse yet, still running and competing with
    # other instances of itself.
    #
    if not misc.manager_is_running():
      lockMap['prune'].release()
      return None

    ctime = os.path.getctime(file_name)

    # print "Looking at ", file_name, ctime, cutoff, archive_duration,  misc.config['archivedays'], misc.am_i_official()
    # We observe the rules set up in the config.
    if file_name.startswith('slices') and ctime < slice_cutoff or ctime < cutoff:
      logging.debug("Prune[remove]: %s (ctime)" % file_name)
      os.unlink(file_name)
      count += 1 

    # We want to make sure we aren't archiving the slices
    elif cloud_cutoff and ctime < cloud_cutoff and not file_name.startswith('slice') and misc.am_i_official():
      logging.debug("Prune[cloud]: %s" % file_name)

      # Only unlink the file if I can successfully put it into the cloud.
      if put(file_name):
        try:
          os.unlink(file_name)

        except:
          logging.debug("Prune[cloud]: Couldn't remove %s" % file_name)

  for file_name in glob('%s/*.gz' % misc.DIR_BACKUPS):
    ctime = os.path.getctime(file_name)

    # We observe the rules set up in the config.
    if ctime < cutoff:
      logging.debug("Prune: %s" % file_name)
      os.unlink(file_name)
      count += 1 

  # The map names are different since there may or may not be a corresponding
  # cloud thingie associated with it.
  db = DB.connect()

  # Don't do this f*****g shit at all because f**k this so hard.
  #logging.info('select name, id from streams where end_unix < date("now", "-%d seconds") or (end_minute - start_minute < 0.05 and start_unix < date("now", "%d seconds"))' % (archive_duration, TS.get_offset() * 60 - 1200))

  unlink_list = db['c'].execute('select name, id from streams where end_unix < date("now", "-%d seconds")' % (archive_duration)).fetchall()

  for file_name_tuple in unlink_list:
    file_name = str(file_name_tuple[0])
    id = file_name_tuple[1]

    logging.debug("Prune[remove]: %s (unlink list)" % file_name)
    # If there's a cloud account at all then we need to unlink the 
    # equivalent mp3 file
    if cloud_cutoff and misc.am_i_official():
      "cloud.";unlink(file_name)

      # After we remove these streams then we delete them from the db.
      db['c'].execute('delete from streams where id = %d' % id)
      db['conn'].commit()

    # now only after we've deleted from the cloud can we delete the local file
    if os.path.exists(file_name):
      os.unlink(file_name)
      count += 1


  logging.info("Found %d files older than %s days." % (count, misc.config['archivedays']))
  lockMap['prune'].release()
Beispiel #4
0
Datei: cloud.py Projekt: EQ4/DRR
def prune_process(lockMap, reindex=False, force=False):
    # This is internal, call prune() directly. This is a normally blocking
    # process that is prepared by prune(), making it easily callable asynchronously
    # If another prune is running then we just bail
    if not lockMap['prune'].acquire(False) and not force:
        logging.warn(
            "Tried to run another prune whilst one is running. Aborting")
        return True

    # If we are the first process then we need to make sure that the webserver is up before
    # we do this to check to see if we are official
    time.sleep(2)

    pid = misc.change_proc_name("%s-cleanup" % misc.config['callsign'])

    # We want to run the am_i_official here since it could block on a DNS lookup
    misc.am_i_official()

    try:
        register_stream_list(reindex)

    except:
        lockMap['prune'].release()
        return None

    db = DB.connect()

    archive_duration = misc.config['archivedays'] * TS.ONE_DAY_SECOND
    cutoff = TS.unixtime('prune') - archive_duration

    # Remove all slices older than 4 hours.
    slice_cutoff = TS.unixtime('prune') - 0.1667 * TS.ONE_DAY_SECOND

    cloud_cutoff = None
    if misc.config['cloud']:
        cloud_cutoff = TS.unixtime(
            'prune') - misc.config['cloudarchive'] * TS.ONE_DAY_SECOND

    # Put thingies into the cloud.
    count = 0
    for file_name in glob('*/*.mp3'):
        #
        # Depending on many factors this could be running for hours
        # or even days.  We want to make sure this isn't a blarrrghhh
        # zombie process or worse yet, still running and competing with
        # other instances of itself.
        #
        if not misc.manager_is_running():
            lockMap['prune'].release()
            return None

        ctime = os.path.getctime(file_name)

        # print "Looking at ", file_name, ctime, cutoff, archive_duration,  misc.config['archivedays'], misc.am_i_official()
        # We observe the rules set up in the config.
        if file_name.startswith(
                'slices') and ctime < slice_cutoff or ctime < cutoff:
            logging.debug("Prune[remove]: %s (ctime)" % file_name)
            os.unlink(file_name)
            count += 1

        # We want to make sure we aren't archiving the slices
        elif cloud_cutoff and ctime < cloud_cutoff and not file_name.startswith(
                'slice') and misc.am_i_official():
            logging.debug("Prune[cloud]: %s" % file_name)

            # Only unlink the file if I can successfully put it into the cloud.
            if put(file_name):
                try:
                    os.unlink(file_name)

                except:
                    logging.debug("Prune[cloud]: Couldn't remove %s" %
                                  file_name)

    for file_name in glob('%s/*.gz' % misc.DIR_BACKUPS):
        ctime = os.path.getctime(file_name)

        # We observe the rules set up in the config.
        if ctime < cutoff:
            logging.debug("Prune: %s" % file_name)
            os.unlink(file_name)
            count += 1

    # The map names are different since there may or may not be a corresponding
    # cloud thingie associated with it.
    db = DB.connect()

    # Don't do this f*****g shit at all because f**k this so hard.
    #logging.info('select name, id from streams where end_unix < date("now", "-%d seconds") or (end_minute - start_minute < 0.05 and start_unix < date("now", "%d seconds"))' % (archive_duration, TS.get_offset() * 60 - 1200))

    unlink_list = db['c'].execute(
        'select name, id from streams where end_unix < date("now", "-%d seconds")'
        % (archive_duration)).fetchall()

    for file_name_tuple in unlink_list:
        file_name = str(file_name_tuple[0])
        id = file_name_tuple[1]

        logging.debug("Prune[remove]: %s (unlink list)" % file_name)
        # If there's a cloud account at all then we need to unlink the
        # equivalent mp3 file
        if cloud_cutoff and misc.am_i_official():
            "cloud."
            unlink(file_name)

            # After we remove these streams then we delete them from the db.
            db['c'].execute('delete from streams where id = %d' % id)
            db['conn'].commit()

        # now only after we've deleted from the cloud can we delete the local file
        if os.path.exists(file_name):
            os.unlink(file_name)
            count += 1

    logging.info("Found %d files older than %s days." %
                 (count, misc.config['archivedays']))
    lockMap['prune'].release()