Esempio n. 1
0
def save_html(data, filename, key):
    path = join(settings.LOG_DIR, key, filename + '.html')
    try:
        write(data, path)
    except IOError:
        mkdir_p(dirname(path))
        write(data, path)
def make_pull_plot_gen(category, misIDRatios, catRatios):
  pull_plot = TH1D(category, category, 6, 0, 6 )
  others_plot = TH1D(category+"others", category+"others", 6, 0, 6 )
  true_plot = TH1D(category+"true", category+"true", 6, 0, 6 )
  bin_names = get_all_containers(category)
  for b in range(1, len(bin_names)+1):
    pull_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    #pull_plot.SetAxisRange(-0.006, 0.006,"Y")
    #others_plot.SetAxisRange(-0.006, 0.006,"Y")
    others_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    true_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    (value, err, err_plus) = catRatios[bin_names[b-1]]
    pull_plot.SetBinContent(b, value)
    pull_plot.SetBinError(b, err)

    other = get_other_component(category, bin_names[b-1])
    (valueO, errO, err0_plus) = misIDRatios[other]
    others_plot.SetBinContent(b, valueO)
    others_plot.SetBinError(b, errO)

    true_plot.SetBinContent(b, misIDRatios[category][0])
    #print bin_names[b-1], value, valueO  
  pull_plot.Add(others_plot, -1)
  c = TCanvas("Plot", "Plot", 1920,1080)
  ROOT.gStyle.SetOptStat(0)
  true_plot.SetLineColor(ROOT.kRed)
  true_plot.SetLineWidth(3)
  true_plot.GetYaxis().SetRangeUser(-0.006, 0.006)
  true_plot.Draw()
  pull_plot.SetLineWidth(3)
  pull_plot.Draw("SAME")
  mydir = "pull_plots_gen/"
  mkdir_p(mydir)
  c.SaveAs("%s/%s_pulls.pdf" % (mydir, category))
  c.SaveAs("%s/%s_pulls.png" % (mydir, category))
Esempio n. 3
0
def build_gdb13_data():
    atom_idxs = {'H': 0, 'C': 1, 'N': 2, 'O': 3, 'F': 4}
    base_path = os.path.join(DATA_BASE_DIR, "gdb13")
    mkdir_p(base_path)

    energies = []
    atom_counts = []
    for name in sorted(os.listdir(os.path.join(base_path, "xyz"))):
        xyz_path = os.path.join(base_path, "xyz", name)
        out_path = xyz_path.replace("xyz", "out")

        natoms = 0
        energy = None
        counts = [0 for _ in atom_idxs]
        with open(xyz_path, 'r') as xyz_f, open(out_path, 'w') as out_f:
            for i, line in enumerate(xyz_f):
                line = line.strip()
                if not i:
                    natoms = int(line)
                elif i == 1:
                    energy = float(line.split()[-3])
                elif i - 2 < natoms:
                    line = line.replace("*^", "e")
                    ele, x, y, z, _ = line.split()
                    counts[atom_idxs[ele]] += 1
                    out_f.write("%s %.8f %.8f %.8f\n" % (ele, float(x), float(y), float(z)))
        energies.append(energy)
        atom_counts.append(counts)
    atom_counts = numpy.matrix(atom_counts)
    atomization = calculate_atomization_energies(atom_counts, numpy.matrix(energies).T)
    atomization *= HARTREE_TO_KCAL
    numpy.savetxt(os.path.join(base_path, "energies.txt"), atomization)
    numpy.savetxt(os.path.join(base_path, "heavy_counts.txt"), atom_counts.sum(1))
Esempio n. 4
0
def saveTime(time):

    dir_ = './.Scores/'
    utils.mkdir_p(dir_)
    f = open(dir_ + '.Scores.txt', 'a')
    f.write(str(time) + '\n')
    f.close()
  def sweep():
    to_check = []

    bioguide = utils.flags().get('bioguide', None)
    if bioguide:
      possibles = [bioguide]
    else:
      possibles = current_bioguide.keys()

    for bioguide in possibles:
      if media_bioguide.get(bioguide, None) is None:
        to_check.append(bioguide)
      elif media_bioguide[bioguide]["social"].get(service, None) is None:
        to_check.append(bioguide)
      else:
        pass

    utils.mkdir_p("cache/social_media")
    writer = csv.writer(open("cache/social_media/%s_candidates.csv" % service, 'w'))
    writer.writerow(["bioguide", "official_full", "website", "service", "candidate", "candidate_url"])

    if len(to_check) > 0:
      email_body = "Social media leads found:\n\n"
      for bioguide in to_check:
        candidate = candidate_for(bioguide)
        if candidate:
          url = current_bioguide[bioguide]["terms"][-1].get("url", None)
          candidate_url = "https://%s.com/%s" % (service, candidate)
          row = [bioguide, current_bioguide[bioguide]['name']['official_full'].encode('utf-8'), url, service, candidate, candidate_url]
          writer.writerow(row)
          print "\tWrote: %s" % candidate
          email_body += ("%s\n" % row)

      if email_enabled:
        utils.send_email(email_body)
def main():
  utils.Initialize()
  e_name_list = gflags.FLAGS.extractors

  new_experiment = experiments.StartNewExperiment(e_name_list)
  experiment_id = new_experiment.GetID()
  utils.mkdir_p(gflags.FLAGS.reports_dir)
  utils.mkdir_p(gflags.FLAGS.models_dir)
  report_loc = path.join(gflags.FLAGS.reports_dir, "%.3d.html" % experiment_id)
  model_loc = path.join(gflags.FLAGS.models_dir, "%.3d.model" % experiment_id)

  print "Experiment ID: %d. Detailed report at %s. Model at %s\n" % (
      experiment_id,
      report_loc,
      model_loc,
  )
  
  cv_data = LoadCVData()
  hd_data = LoadHDData()

  new_experiment.RunCrossValidation(cv_data)

  model = models.BuildModel(e_name_list, cv_data)
  model.Save(model_loc)
  hd_result = model.EvaluateOn(hd_data)
  new_experiment.RecordHeldoutDataEval(hd_result)

  new_experiment.Save()
  new_experiment.PrintSummary()
  new_experiment.ExportReport(report_loc)
def process_clip(clip_name, paths, img_type, negative_images, overwrite=True):
    # overwrite: overwrite the training of the FFLD model.
    print(clip_name)
    frames_path = paths['clips'] + frames + clip_name + sep
    if not check_path_and_landmarks(frames_path, clip_name, paths['in_bb'] + clip_name + sep):  # check that paths, landmarks exist
        return
    list_frames = sorted(os.listdir(frames_path))
    save_model = paths['out_model'] + clip_name + '.model'
    if (not os.path.exists(save_model)) or overwrite:
        # build the detector
        training_pos = load_images(list_frames, frames_path, paths['in_bb'], clip_name, max_images=400)
        if len(training_pos) == 0:
            print('No positives found for the clip {}, skipping it.'.format(clip_name))
            return
        ps_model = train_ffld2_detector(training_pos, negative_images, n_components=1, n_relabel=6)
        ps_model.save(save_model)
    else:
        print('The model {} already exists and was loaded from disk.'.format(save_model))
        ps_model = load_model(save_model)
    global detector
    detector = FFLD2Detector(ps_model)

    p_det_bb = mkdir_p(paths['out_bb'] + clip_name + sep)
    p_det_landm = mkdir_p(paths['out_lns'] + clip_name + sep)
    clip = Clip(clip_name, paths['clips'], frames, write_ln=[p_det_bb, p_det_landm])
    # TODO: Try parallel model
    [predict_in_frame(frame_name, clip, img_type) for frame_name in list_frames]
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--image_list', required=True)
    parser.add_argument('--config', required=True)
    parser.add_argument('--dump_prefix', required=True)
    parser.add_argument('--output_dir', required=True)
    args = parser.parse_args()

    conf_mod = imp.load_source('config', args.config)
    config = conf_mod.get()

    model = config['model']
    utils.load_model(model, args.dump_prefix)

    X, _ = conf_mod.get_data(args.image_list)

    utils.mkdir_p(args.output_dir)
    image_list = utils.read_image_list(args.image_list)

    logger.info('compiling model ...')
    model.compile(loss='mean_squared_error', optimizer=Adam())

    for x, (input_path, _) in ProgressBar()(zip(X, image_list)):
        y = model.predict(np.array([x], dtype='float32'),
                               batch_size=1, verbose=False)
        img = np.round(y.reshape(y.shape[2:]) * 255.0).astype('uint8')

        # FIXME: we assume that basenames of images are distinct
        fname = os.path.basename(input_path)
        output_path = os.path.join(args.output_dir, fname)
        cv2.imwrite(output_path, img)
 def test_mkdir_p(self):
     d = mkdtemp()
     path = os.path.join(d, 'a/b/c')
     mkdir_p(path)
     self.assertTrue(os.path.isdir(path))
     mkdir_p(path)  # Already exists test
     rmtree(d)
Esempio n. 10
0
    def create_test_case(self):
        """
        Create a test case.
        """
        testyml_template = default_testyml_template.replace(
            "%role", self.normalized_role)
        testyml_template = testyml_template.replace(
            "%year", str(date.today().year))
        testyml_template = testyml_template.replace(
            "%author", self.config["author_name"])
        testyml_template = testyml_template.replace(
            "%email", self.config["author_email"])

        utils.mkdir_p(os.path.join(self.output_path,
                                   "tests", "inventory", "group_vars"))

        utils.mkdir_p(os.path.join(self.output_path,
                                   "tests", "inventory", "host_vars"))

        hosts = "placeholder_fqdn\n"
        utils.string_to_file(os.path.join(self.output_path,
                                          "tests", "inventory", "hosts"),
                             hosts)

        test_file = os.path.join(self.output_path, "tests", "test")
        utils.string_to_file(test_file, testyml_template)

        os.chmod(test_file, 0755)
Esempio n. 11
0
 def user_login(self, user):
     """
     Called immediately after a user authenticates successfully.  Saves
     session information in the user's directory.  Expects *user* to be a
     dict containing a 'upn' value representing the username or
     userPrincipalName. e.g. '[email protected]' or just 'someuser'.  Any additional
     values will be attached to the user object/cookie.
     """
     logging.debug("user_login(%s)" % user["upn"])
     user.update(additional_attributes(user))
     # Make a directory to store this user's settings/files/logs/etc
     user_dir = os.path.join(self.settings["user_dir"], user["upn"])
     if not os.path.exists(user_dir):
         logging.info(_("Creating user directory: %s" % user_dir))
         mkdir_p(user_dir)
         os.chmod(user_dir, 0o700)
     session_file = os.path.join(user_dir, "session")
     session_file_exists = os.path.exists(session_file)
     if session_file_exists:
         session_data = open(session_file).read()
         try:
             session_info = tornado.escape.json_decode(session_data)
         except ValueError:  # Something wrong with the file
             session_file_exists = False  # Overwrite it below
     if not session_file_exists:
         with open(session_file, "w") as f:
             # Save it so we can keep track across multiple clients
             session_info = {"session": generate_session_id()}
             session_info.update(user)
             session_info_json = tornado.escape.json_encode(session_info)
             f.write(session_info_json)
     self.set_secure_cookie("gateone_user", tornado.escape.json_encode(session_info))
def download_cfo(options):
    if options.get('loglevel', None):
        log.setLevel(options['loglevel'])

    OUT_DIR = os.path.join(CACHE_DIR, 'cfo')
    if not os.path.exists(OUT_DIR):
        mkdir_p(OUT_DIR)

    base_url = 'https://www.campaignfinanceonline.state.pa.us/pages/CFAnnualTotals.aspx'

    def _get_response_loc_pair(dl_info):
        filer_id = dl_info
        loc = os.path.join(OUT_DIR, '{}.html'.format(filer_id))
        response = requests.get(base_url, params={'Filer': filer_id})
        return (response, loc)

    filer_ids = set([])

    for loc in iglob(os.path.join(
            CACHE_DIR, 'dos', '*', '*', '[fF]iler.[Tt]xt')):
        with open(loc, 'r') as fin:
            for row in csv.reader(fin):
                if row[0]:
                    filer_ids.add(row[0])

    download_all(list(filer_ids), _get_response_loc_pair, options)
Esempio n. 13
0
def processResults(df, features, output_path, filename):
    TeamObjects = {TeamsDict['Team_Id'][k] : {'Team_Name' : TeamsDict['Team_Name'][k]} for k in TeamsDict['Team_Id']}
    for season in range(2003,2016):
        for k in TeamObjects:
            for f in features:
                TeamObjects[k][f] = 0
            TeamObjects[k]['GameCount'] = 0

        for index, game in df[df.Season == season].iterrows():
            d = game.to_dict()
            Wteam = d['Wteam']
            Lteam = d['Lteam']

            for f in features:
                if f.startswith('W'):
                    TeamObjects[Wteam][f] += d[f.replace('Avg', '')]
                if f.startswith('L'):
                    TeamObjects[Lteam][f] += d[f.replace('Avg', '')]

            TeamObjects[Wteam]['GameCount'] += 1
            TeamObjects[Lteam]['GameCount'] += 1

        for k in TeamObjects:
            for f in features:
                if TeamObjects[k]['GameCount'] > 0:
                    TeamObjects[k][f] /= TeamObjects[k]['GameCount']

        TeamStats = pandas.DataFrame.from_dict(TeamObjects, orient='index')
        mkdir_p(output_path)
        TeamStats.to_csv(output_path + filename + str(season) + '.csv')
        print('Wrote out ' + output_path + filename + str(season) + '.csv')
Esempio n. 14
0
 def user_login(self, user):
     """
     Called immediately after a user authenticates successfully.  Saves
     session information in the user's directory.  Expects *user* to be a
     string containing the username or userPrincipalName. e.g. '[email protected]'
     or just 'someuser'.
     """
     logging.debug("user_login(%s)" % user)
     # Make a directory to store this user's settings/files/logs/etc
     user_dir = os.path.join(self.settings['user_dir'], user)
     if not os.path.exists(user_dir):
         logging.info(_("Creating user directory: %s" % user_dir))
         mkdir_p(user_dir)
         os.chmod(user_dir, 0o700)
     session_file = os.path.join(user_dir, 'session')
     session_file_exists = os.path.exists(session_file)
     if session_file_exists:
         session_data = open(session_file).read()
         try:
             session_info = tornado.escape.json_decode(session_data)
         except ValueError: # Something wrong with the file
             session_file_exists = False # Overwrite it below
     if not session_file_exists:
         with open(session_file, 'w') as f:
             # Save it so we can keep track across multiple clients
             session_info = {
                 'upn': user, # FYI: UPN == userPrincipalName
                 'session': generate_session_id()
             }
             session_info_json = tornado.escape.json_encode(session_info)
             f.write(session_info_json)
     self.set_secure_cookie(
         "gateone_user", tornado.escape.json_encode(session_info))
Esempio n. 15
0
def backup(path, password_file=None):
    """
    Replaces the contents of a file with its decrypted counterpart, storing the
    original encrypted version and a hash of the file contents for later
    retrieval.
    """
    vault = VaultLib(get_vault_password(password_file))
    with open(path, 'r') as f:
        encrypted_data = f.read()

        # Normally we'd just try and catch the exception, but the
        # exception raised here is not very specific (just
        # `AnsibleError`), so this feels safer to avoid suppressing
        # other things that might go wrong.
        if vault.is_encrypted(encrypted_data):
            decrypted_data = vault.decrypt(encrypted_data)

            # Create atk vault files
            atk_path = os.path.join(ATK_VAULT, path)
            mkdir_p(atk_path)
            # ... encrypted
            with open(os.path.join(atk_path, 'encrypted'), 'wb') as f:
                f.write(encrypted_data)
            # ... hash
            with open(os.path.join(atk_path, 'hash'), 'wb') as f:
                f.write(hashlib.sha1(decrypted_data).hexdigest())

            # Replace encrypted file with decrypted one
            with open(path, 'wb') as f:
                f.write(decrypted_data)
def actually_create_android_project(package_name, sdk_version, java_package_name, is_library):
    path = os.path.join(os.getcwd(), package_name.lower())
    console.pretty_println("\nCreating android project ", console.bold)
    console.pretty_print("  Name      : ", console.cyan)
    console.pretty_println("%s" % package_name, console.yellow)
    console.pretty_print("  Sdk Ver   : ", console.cyan)
    console.pretty_println("%s" % sdk_version, console.yellow)
    console.pretty_print("  Java Name : ", console.cyan)
    console.pretty_println("%s" % java_package_name, console.yellow)
    if is_library:
        console.pretty_print("  Library   : ", console.cyan)
        console.pretty_println("yes\n", console.yellow)
        cmd = ['android', 'create', 'lib-project', '-n', package_name, '-p', path, '-k', java_package_name, '-t', 'android-' + sdk_version, ]
    else:
        activity_name = utils.camel_case(package_name)
        console.pretty_print("  Activity  : ", console.cyan)
        console.pretty_println("%s\n" % activity_name, console.yellow)
        cmd = ['android', 'create', 'project', '-n', package_name, '-p', path, '-k', java_package_name, '-t', 'android-' + sdk_version, '-a', activity_name]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError:
        raise subprocess.CalledProcessError("failed to create android project.")
    # This is in the old form, let's shovel the shit around to the new form
    utils.mkdir_p(os.path.join(path, 'src', 'main', 'java'))
    os.remove(os.path.join(path, 'local.properties'))
    os.remove(os.path.join(path, 'project.properties'))
    os.remove(os.path.join(path, 'ant.properties'))
    os.remove(os.path.join(path, 'proguard-project.txt'))
    os.remove(os.path.join(path, 'build.xml'))
    os.rmdir(os.path.join(path, 'bin'))
    os.rmdir(os.path.join(path, 'libs'))
    shutil.move(os.path.join(path, 'AndroidManifest.xml'), os.path.join(path, 'src', 'main'))
    shutil.move(os.path.join(path, 'res'), os.path.join(path, 'src', 'main'))
    if not is_library:
        shutil.move(os.path.join(path, 'src', java_package_name.split('.')[0]), os.path.join(path, 'src', 'main', 'java'))
Esempio n. 17
0
    def setup(self):
        """
        Create a working directory and some test files
        """
        self.working_dir = tempfile.mkdtemp()
        self.file_contents = collections.OrderedDict.fromkeys([
            'file.test',
            '1/file.test',
            '2/2/file.test',
        ])
        self.file_timestamps = self.file_contents.copy()

        # create a key for the tests
        self.key = _create_key(None, write=False)

        # setup files in subdirectory
        for path in self.file_contents.keys():
            # create file content
            self.file_contents[path] = str(uuid.uuid4())

            abspath = os.path.join(self.working_dir, path)

            # create subdirs as necessary
            mkdir_p(os.path.dirname(abspath))

            # create test file in dir
            with open(abspath, 'w') as f:
                f.write(self.file_contents[path])

            # record file creation time
            self.file_timestamps[path] = os.stat(abspath).st_ctime
Esempio n. 18
0
def main():
    """
    Main entry point for execution as a program (instead of as a module).
    """

    args = parse_args()
    completed_classes = []

    mkdir_p(PATH_CACHE, 0o700)
    if args.clear_cache:
        shutil.rmtree(PATH_CACHE)
    if args.on_demand:
        logging.warning(
            "--on-demand option is deprecated and is not required"
            " anymore. Do not use this option. It will be removed"
            "in the future."
        )

    for class_name in args.class_names:
        try:
            logging.info("Downloading class: %s", class_name)
            if download_class(args, class_name):
                completed_classes.append(class_name)
        except requests.exceptions.HTTPError as e:
            logging.error("HTTPError %s", e)
        except ClassNotFound as cnf:
            logging.error("Could not find class: %s", cnf)
        except AuthenticationFailed as af:
            logging.error("Could not authenticate: %s", af)

    if completed_classes:
        logging.info("Classes which appear completed: " + " ".join(completed_classes))
def make_pull_plot(category, misIDRatios, catRatios, datastring, fitname, fittype):
  pull_plot = TH1D(category, category, 6, 0, 6 );
  others_plot = TH1D(category+"others", category+"others", 6, 0, 6 );
  bin_names = get_all_containers(category)
  for b in range(1, len(bin_names)+1):
    pull_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    others_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    (value, err) = catRatios[get_bin_nr_composite(bin_names[b-1])]
    pull_plot.SetBinContent(b, value)
    pull_plot.SetBinError(b, err)

    other = get_other_component(category, bin_names[b-1])
    (valueO, errO) = misIDRatios[get_bin_nr_single(other)]
    others_plot.SetBinContent(b, valueO)
    others_plot.SetBinError(b, errO)
    #print bin_names[b-1], value, valueO  
  pull_plot.Add(others_plot, -1)
  c = TCanvas("Plot", "Plot", 800,600)
  ROOT.gStyle.SetOptStat(0)
  pull_plot.Draw()
  if len(fittype) == 0: fittype = "histograms"
  mydir = "pull_plots/%s/%s/%s/" % (fitname, fittype, datastring)
  mkdir_p(mydir)
  c.SaveAs("%s/%s_pulls.pdf" % (mydir, category))
  c.SaveAs("%s/%s_pulls.png" % (mydir, category))
  def sweep():
    to_check = []

    bioguide = utils.flags().get('bioguide', None)
    if bioguide:
      possibles = [bioguide]
    else:
      possibles = current_bioguide.keys()

    for bioguide in possibles:
      if media_bioguide.get(bioguide, None) is None:
        to_check.append(bioguide)
      elif media_bioguide[bioguide]["social"].get(service, None) is None:
        to_check.append(bioguide)
      else:
        pass

    utils.mkdir_p("cache/social_media")
    writer = csv.writer(open("cache/social_media/%s_candidates.csv" % service, 'w'))
    writer.writerow(["bioguide", "official_full", "website", "service", "candidate"])

    for bioguide in to_check:
      candidate = candidate_for(bioguide)
      if candidate:
        url = current_bioguide[bioguide]["terms"][-1].get("url", None)
        writer.writerow([bioguide, current_bioguide[bioguide]['name']['official_full'], url, service, candidate])
        print "\tWrote: %s" % candidate
Esempio n. 21
0
 def user_login(self, user):
     """
     Called immediately after a user authenticates successfully.  Saves
     session information in the user's directory.  Expects *user* to be a
     string containing the username or userPrincipalName. e.g. '[email protected]'
     or just 'someuser'.
     """
     # Make a directory to store this user's settings/files/logs/etc
     user_dir = self.settings['user_dir'] + "/" + user
     logging.info("Creating user directory: %s" % user_dir)
     mkdir_p(user_dir)
     os.chmod(user_dir, 0700)
     session_file = user_dir + '/session'
     if os.path.exists(session_file):
         session_data = open(session_file).read()
         session_info = tornado.escape.json_decode(session_data)
     else:
         with open(session_file, 'w') as f:
             # Save it so we can keep track across multiple clients
             session_info = {
                 'go_upn': user, # FYI: UPN == userPrincipalName
                 'go_session': generate_session_id()
             }
             session_info_json = tornado.escape.json_encode(session_info)
             f.write(session_info_json)
     self.set_secure_cookie("user", tornado.escape.json_encode(session_info))
Esempio n. 22
0
    def create_skeleton(self):
        """
        Create the role's directory and file structure.
        """
        utils.string_to_file(os.path.join(self.output_path, "VERSION"),
                             "master\n")

        for folder in c.ANSIBLE_FOLDERS:
            create_folder_path = os.path.join(self.output_path, folder)
            utils.mkdir_p(create_folder_path)

            mainyml_template = default_mainyml_template.replace(
                "%role_name", self.role_name)
            mainyml_template = mainyml_template.replace(
                "%values", folder)

            out_path = os.path.join(create_folder_path, "main.yml")

            if folder not in ("templates", "meta", "tests", "files"):
                utils.string_to_file(out_path, mainyml_template)

            if folder == "meta":
                utils.create_meta_main(out_path,
                                       self.config, self.role_name,
                                       self.options.galaxy_categories)
Esempio n. 23
0
def main():
    """
    Main entry point for execution as a program (instead of as a module).
    """

    args = parseArgs()
    completed_classes = []

    mkdir_p(PATH_CACHE, 0o700)
    if args.clear_cache:
        shutil.rmtree(PATH_CACHE)

    for class_name in args.class_names:
        try:
            logging.info('Downloading class: %s', class_name)
            if download_class(args, class_name):
                completed_classes.append(class_name)
        except requests.exceptions.HTTPError as e:
            logging.error('HTTPError %s', e)
        except ClassNotFound as cnf:
            logging.error('Could not find class: %s', cnf)
        except AuthenticationFailed as af:
            logging.error('Could not authenticate: %s', af)

    if completed_classes:
        logging.info(
            "Classes which appear completed: " + " ".join(completed_classes))
Esempio n. 24
0
def download_syllabus_icourse163(session, leclist, path = '', overwrite = False):

    headers = {
                'Accept':'*/*',
                'Accept-Encoding':'gzip, deflate, sdch',
                'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
                'Connection':'keep-alive',
                'Host':'v.stu.126.net', #*
                'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
                'X-Requested-With':'ShockwaveFlash/15.0.0.239',
               }

    session.headers.update(headers)

    retry_list = []
    for week in leclist:
        cur_week = week[0]
        lessons = week[1]
        for lesson in lessons:
            cur_lesson = lesson[0]
            lectures = lesson[1]
            cur_week = clean_filename(cur_week)
            cur_lesson = clean_filename(cur_lesson)
            dir = os.path.join(path, cur_week, cur_lesson)
            if not os.path.exists(dir):
                mkdir_p(dir)

            for (lecnum, (lecture_url, lecture_name)) in enumerate(lectures):
                lecture_name = clean_filename(lecture_name)
                filename = os.path.join(dir,"%02d_%s.%s"%(lecnum+1, lecture_name, lecture_url[-3:]))
                print (filename)
                print (lecture_url)
                try:
                    resume_download_file(session, lecture_url, filename, overwrite )
                except Exception as e:
                    print(e)
                    print('Error, add it to retry list')
                    retry_list.append((lecture_url, filename))

    retry_times = 0
    while len(retry_list) != 0 and retry_times < 3:
        print('%d items should be retried, retrying...' % len(retry_list))
        tmp_list = [item for item in retry_list]
        retry_times += 1
        for (url, filename) in tmp_list:
            try:
                print(url)
                print(filename)
                resume_download_file(session, url, filename, overwrite )
            except Exception as e:
                print(e)
                print('Error, add it to retry list')
                continue

            retry_list.remove((url, filename)) 
    
    if len(retry_list) != 0:
        print('%d items failed, please check it' % len(retry_list))
    else:
        print('All done.')
Esempio n. 25
0
    def read_cache(self):
        """
        Read cache object inside the .bookbuilder/cache_object.txt.

        Returns:
            None if cache_object.txt doesn't exist
            cache_object of type dict if it does
        """
        # check whether .bookbuilder folder exists
        # and initialise it if it doesn't
        if not os.path.exists('.bookbuilder'):
            print("Creating .bookbuilder folder")
            mkdir_p('.bookbuilder')

        cache_object_path = os.path.join('.bookbuilder', 'cache_object.txt')

        if not os.path.exists(cache_object_path):
            # create one if it doesn't exist
            cache_object = self.create_cache_object()

            return cache_object
        else:
            with open(cache_object_path, 'r') as cop:
                copcontent = cop.read()
                if len(copcontent) == 0:
                    cache_object = self.create_cache_object()
                else:
                    cache_object = ast.literal_eval(copcontent)

                return cache_object
Esempio n. 26
0
def anonymize_existing(bids_warehouse,anonmap,bids_log):
    '''
    Name: anonymize_existing
    Description: This function will anonymized BIDSified data in the non-anonymized directory.
    Arguments:
    ================================================================================================
    bids_warehouse : string
        A full path to the BIDS warehouse.
    anonmap : dict (optional)
        A dictionary mapping URSIs to anonymous IDs.  Used if anonymization is to occur. URSIs are keys, anonymous IDs are values.
    bids_log : logger
        A logger for the image to BIDS conversion.
    '''
    nonanon_dir=os.path.join(bids_warehouse,'Non-anonymized')
    anon_dir=os.path.join(bids_warehouse,'Anonymized')
    for nonanon_root, dirnames, filenames in os.walk(nonanon_dir):
        for filename in filenames:
            participants_tsv=False
            nonanon_file = os.path.join(nonanon_root,filename)
            ursi=re.findall('M[0-9]{8}',nonanon_file)
            if ursi:
                ursi=ursi[0]
            elif 'participants.tsv' in nonanon_file:
                participants_tsv=True
                anon_file=os.path.join(anon_dir,'participants.tsv')
            else:
                bids_log.info('Could not find URSI in file %s.  (Probably an inherited JSON)' % nonanon_file)
                continue
            if not participants_tsv:
                if ursi not in anonmap.keys():
                    bids_log.info('URSI %s not in anonymization map.   Skipping...' % ursi)
                    continue
                anon_root = nonanon_root.replace(ursi,anonmap[ursi])
                anon_root = anon_root.replace(nonanon_dir,anon_dir)
                anon_file = nonanon_file.replace(ursi,anonmap[ursi])
                anon_file = anon_file.replace(nonanon_dir,anon_dir)
                mkdir_p(anon_root)
            if not os.path.isfile(anon_file):
                if '.nii.gz' in nonanon_file:
                    try:
                        shutil.copy(nonanon_file,anon_file)
                    except:
                        bids_log.info('Could not copy %s' % nonanon_file)
                else:
                    try:
                        with open(nonanon_file,'rU') as nonanon_f:
                            with open(anon_file,'w') as anon_f:
                                for line in nonanon_f:
                                    ursi=re.findall('M[0-9]{8}',line)
                                    if ursi:
                                        ursi=ursi[0]
                                        if ursi in anonmap.keys():
                                            anon_f.write(line.replace(ursi,anonmap[ursi]))
                                    else:
                                        anon_f.write(line)
                    except:
                        bids_log.info('Could not copy %s' % nonanon_file)
            else:
                bids_log.info('%s is already anonymized' % nonanon_file)
def extract_cfo(options):
    if options.get('loglevel', None):
        log.setLevel(options['loglevel'])

    OUT_DIR = os.path.join(ORIG_DIR, 'cfo')
    if not os.path.exists(OUT_DIR):
        mkdir_p(OUT_DIR)

    CFO_CACHE = os.path.join(CACHE_DIR, 'cfo')

    html_parser = etree.HTMLParser()

    def _chunks(l, n):
        """ Yield successive n-sized chunks from l.
        """
        for i in xrange(0, len(l), n):
            yield l[i:i+n]

    def _parse_data_tables(d):
        return pd.io.html.read_html(etree.tostring(d), header=1, index_col=0)

    def _parse_year_table(y):
        return y.xpath('.//tr[1]/td[2]/span')[0].text

    def _extract_tables(pg_html):
        all_tables_container = pg_html.xpath(
            "//div[@id='ctl00_ContentPlaceHolder1_divCFSummary']")[0]
        summary_tables = {_parse_year_table(y): _parse_data_tables(d)[0]
                          for y, d in
                          _chunks(all_tables_container.xpath("table"), 2)}
        return summary_tables

    for loc in iglob(os.path.join(CFO_CACHE, '*.html')):
        log.debug('opening {l}'.format(l=loc))
        filer_id = os.path.splitext(os.path.split(loc)[1])[0]
        with open(loc, 'r') as fin:
            try:
                pg_html = etree.parse(fin, parser=html_parser)
                tables = _extract_tables(pg_html)
            except Exception as e:
                log.error('parsing file {l} failed:'.format(l=loc))
                log.error(e)

            try:
                for year, table in tables.iteritems():
                    if year:
                        output_dir = os.path.join(OUT_DIR, year)
                        if not os.path.exists(output_dir):
                            mkdir_p(output_dir)
                        output_loc = os.path.join(OUT_DIR, year,
                                                  '{}.json'.format(filer_id))
                        table.dropna(axis=1, how='all').to_json(
                            path_or_buf=output_loc, orient='index')
                    else:
                        log.debug('{l} contained {y} as a year?'.format(
                            l=loc, y=year))
            except Exception as e:
                log.error('reading table dict {l} failed:'.format(l=loc))
                log.error(e)
 def _get_response_loc_pair(dl_info):
     year, period, filename, filesize, atag_url = dl_info
     loc_dir = os.path.join(OUT_DIR, year, period)
     if not os.path.exists(loc_dir):
         mkdir_p(loc_dir)
     loc = os.path.join(loc_dir, filename)
     response = requests.get(atag_url, stream=True)
     return (response, loc)
Esempio n. 29
0
 def safe_writeScreenshotDescriptionFile(self, out_fname):
     """
     writes screenshot descr file in a safe mode. any problems are reported via warning
     :param out_fname: {str}
     :return: None
     """
     mkdir_p(dirname(out_fname))
     self.writeScreenshotDescriptionFile(out_fname)
def create_rosjava_project_common(args, template_directory):
    project_name = args.name[0]
    console.pretty_println("\nCreating rosjava project ", console.bold)
    console.pretty_print("  Name      : ", console.cyan)
    console.pretty_println("%s" % project_name, console.yellow)
    utils.mkdir_p(os.path.join(os.getcwd(), project_name.lower()))
    # This is in the old form, let's shovel the shit around to the new form
    create_gradle_package_files(args, template_directory)
    add_to_root_gradle_settings(args.name[0])