Пример #1
0
def my_hook(d):
    if d['status'] == 'downloading':
        speed_info = ""
        
        if 'filename' in d:
            filename = os.path.basename(d['filename'])
        else:
            filename = ""
    
        if '_eta_str' in d and '_speed_str' in d:
            speed_info = " (" + d['_speed_str'] + " ETA: " + d['_eta_str'] + ")"
        filename = filename.encode('ascii', 'ignore').decode('ascii')
        dz.begin("Downloading " + filename + speed_info + "...")
        total_bytes = 0
        
        if 'downloaded_bytes' in d:
            if 'total_bytes' in d:
                total_bytes = d['total_bytes']
            elif 'total_bytes_estimate' in d:
                total_bytes = d['total_bytes_estimate']
            
            percent = int(100 * d['downloaded_bytes'] / total_bytes)
            if percent > 0:
                utils.set_determinate_progress(True)
                utils.set_progress_percent(percent)
    
    if d['status'] == 'finished':
        utils.set_determinate_progress(False)
        utils.reset_progress()
        print('Download complete')
Пример #2
0
def my_hook(d):
    if d['status'] == 'downloading':
        speed_info = ""

        if 'filename' in d:
            filename = os.path.basename(d['filename'])
        else:
            filename = ""

        if '_eta_str' in d and '_speed_str' in d:
            speed_info = " (" + d['_speed_str'] + " ETA: " + d['_eta_str'] + ")"
        filename = filename.encode('ascii', 'ignore').decode('ascii')
        dz.begin("Downloading " + filename + speed_info + "...")
        total_bytes = 0

        if 'downloaded_bytes' in d:
            if 'total_bytes' in d:
                total_bytes = d['total_bytes']
            elif 'total_bytes_estimate' in d:
                total_bytes = d['total_bytes_estimate']

            percent = int(100 * d['downloaded_bytes'] / total_bytes)
            if percent > 0:
                utils.set_determinate_progress(True)
                utils.set_progress_percent(percent)

    if d['status'] == 'finished':
        utils.set_determinate_progress(False)
        utils.reset_progress()
        print('Download complete')
Пример #3
0
def train_models():
    if len(utils.CATEGORIES) == 0 or utils.not_enough_images():
        err = utils.get_not_enough_images_error()
        return jsonify(error=err)

    utils.delete_all_models()
    utils.set_maybe_old(True)
    maybe_update_models()

    try:
        regression_train.train()
        cnn_train.train()
    except:
        err = "Unknown error."
        utils.delete_all_models()
        return jsonify(error=err)

    if utils.train_should_stop():
        utils.delete_all_models()
        utils.train_should_stop(False)
    else:
        utils.set_maybe_old(False)

    utils.reset_progress()

    return "ok"
Пример #4
0
def download_url(url):
    regex = re.compile(
        r'^(?:http)s?://'  # http:// or https://
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  #domain...
        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
        r'(?::\d+)?'  # optional port
        r'(?:/?|[/?]\S+)$',
        re.IGNORECASE)

    if not regex.match(url):
        dz.fail("Not a valid video URL")

    dz.begin("Checking youtube-dl library is up to date...")
    utils.set_determinate_progress(False)

    updater.update_youtubedl()

    dz.begin("Preparing to download video...")
    utils.set_determinate_progress(False)
    utils.reset_progress()

    # Put ffmpeg in PATH for merging videos audio and video
    if 'apple_silicon' in os.environ:
        os.environ["PATH"] += os.pathsep + os.path.join(
            os.getcwd(), 'ffmpeg-arm')
    else:
        os.environ["PATH"] += os.pathsep + os.path.join(os.getcwd(), 'ffmpeg')

    # Download URL from clipboard
    sys.path.append("yt-dlp")

    from yt_dlp import YoutubeDL

    ydl_opts = {
        'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
        'outtmpl': os.path.join(os.environ['EXTRA_PATH'], '%(title)s.%(ext)s'),
        'logger': MyLogger(),
        'progress_hooks': [my_hook]
    }

    try:
        with YoutubeDL(ydl_opts) as ydl:
            ydl.download([url])
    except Exception as e:
        print(traceback.format_exc())
        dz.error("Video Download Failed",
                 "Downloading video failed with the error:\n\n" + e.message)

    dz.finish("Video Download Complete")
    dz.url(False)
Пример #5
0
def download_url(url):
    regex = re.compile(
            r'^(?:http)s?://' # http:// or https://
            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
            r'(?::\d+)?' # optional port
            r'(?:/?|[/?]\S+)$', re.IGNORECASE)
    
    if not regex.match(url):
        dz.fail("Not a valid video URL")
    
    dz.begin("Checking youtube-dl library is up to date...")
    utils.set_determinate_progress(False)

    updater.update_youtubedl()
    
    dz.begin("Preparing to download video...")
    utils.set_determinate_progress(False)
    utils.reset_progress()
    
    # Put ffmpeg in PATH for merging videos audio and video
    os.environ["PATH"] += os.pathsep + os.path.join(os.getcwd(), 'ffmpeg')
    
    # Download URL from clipboard
    sys.path.append("youtube-dl")
    import youtube_dl
    
    ydl_opts = {
        'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
        'outtmpl': os.path.join(os.environ['EXTRA_PATH'], '%(title)s.%(ext)s'),
        'logger': MyLogger(),
        'progress_hooks': [my_hook]
    }
    
    try:
        with youtube_dl.YoutubeDL(ydl_opts) as ydl:
            ydl.download([url])
    except Exception as e:
        print(traceback.format_exc())
        dz.error("Video Download Failed", "Downloading video failed with the error:\n\n" + e.message)
        
    dz.finish("Video Download Complete")
    dz.url(False)
Пример #6
0
import time
import sys

import config
import utils
    
if __name__ == '__main__':
  utils.init()
  if len(sys.argv) != 3:
    print 'Usage: %s <team_username> <problem_alias>' % sys.argv[0]
    sys.exit(1)
  
  team_username = sys.argv[1]
  problem_alias = sys.argv[2]
  
  utils.reset_progress(True)
    
  task = utils.call(action='fetch_run', team_username=team_username, problem_alias=problem_alias)
  if not task['success']:
    raise Exception('Failed to fetch run.')
  
  print 'Grading run_id %s (team %s, problem %s) of type %s... ' % (task['run_id'], task['team_username'], task['alias'], task['problem_type']),
  
  module = utils.import_module(task['contest_type'], task['problem_type'])
  
  for key in ['run_metadata', 'problem_metadata', 'division_metadata']:
    task[key] = json.loads(task[key])
  
  q = multiprocessing.Queue()
  grader = multiprocessing.Process(target=module.grade, args=(q, task, True))
  grader.start()
Пример #7
0
    return 'judge_id %d, contest_id %d' % (self.judge_id, self.contest_id)

if __name__ == '__main__':
  utils.init()
  judge = AutoJudge()
  print time.strftime('[%H:%M:%S]:', time.localtime()),
  print 'Initialized judge to %s' % judge
  
  while True:
    print time.strftime('[%H:%M:%S]:', time.localtime()),
    task = judge.fetch_task()
    task_type = task['task_type']
    if task_type == 'grade':
      task['run_metadata'] = json.loads(task['run_metadata'])
      print 'Grading run_id %s (team %s, problem %s) of type %s... ' % (task['run_id'], task['team_username'], task['alias'], task['problem_type']),
      utils.reset_progress(False)
      
      problem_metadata, division_metadata = judge.get_cached_metadata(task['problem_id'], task['division_id'], task['problem_metadata_hash'], task['division_metadata_hash'])
      if problem_metadata is None or division_metadata is None:
        utils.progress('Refreshing metadata')
        problem_metadata, division_metadata = judge.update_cached_metadata(task['problem_id'], task['division_id'], task['problem_metadata_hash'], task['division_metadata_hash'])
      
      task['problem_metadata'] = problem_metadata
      task['division_metadata'] = division_metadata
      
      module = utils.import_module(judge.contest_type, task['problem_type'])
      q = multiprocessing.Queue()
      grader = multiprocessing.Process(target=module.grade, args=(q, task, False))
      grader.start()
      result = q.get()
      grader.join()
Пример #8
0
    return word


def clean(txt, stem=False, spell=False):
    # print txt
    txt = txt.lower().strip()
    txt = txt.replace('^p', '')
    txt = re.sub('[%s]' % re.escape(string.punctuation + u'“”«»–—―◦℅™•№▪'), ' ', txt)
    txt = u' '.join([convert_word(i, stem, spell) for i in tokenizer.tokenize(txt) if i not in russtop])

    # print txt
    return txt


if __name__ == '__main__':
    if len(sys.argv) != 3:
        print "Usage " + sys.argv[0] + " file cleanfile"
        sys.exit(1)

    train_file = sys.argv[1]
    clean_filename = sys.argv[2]

    prev_l = ''

    utils.reset_progress()
    with codecs.open(clean_filename, 'w', 'utf-8') as fw:
        for parts in utils.read_train(train_file):
            parts[3] = clean(parts[3], False, False)
            parts[4] = clean(parts[4], False, False)
            fw.write('\t'.join(parts))
            utils.log_progress()