Esempio n. 1
0
def load_midb(src):
    session = model.SESSION()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    cols = [c.name for c in model.MidB.__table__.columns]
    int_cols = [
        c.name for c in model.MidB.__table__.columns
        if isinstance(c.type, sa.types.Integer)
    ]
    cnt = collections.Counter()
    for row in reader:
        cnt[(row['disp_num'], row['ccode'])] += 1
        row['spell_num'] = cnt[(row['disp_num'], row['ccode'])]
        start_date = utils.daterng(*(_int(row[k])
                                     for k in ('st_year', 'st_mon', 'st_day')))
        row['st_date_min'], row['st_date_min'] = start_date
        end_date = utils.daterng(*(_int(row[k])
                                   for k in ('end_year', 'end_mon',
                                             'end_day')))
        row['end_date_min'], row['end_date_min'] = end_date
        ## set -9 to NULL
        for k in int_cols:
            row[k] = _int(row[k])
        session.add(model.MidB(**utils.subset(row, cols)))
    session.commit()
Esempio n. 2
0
 def load_file(src):
     reader = csv2.DictReader(src, encoding='latin-1')
     reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
     for row in reader:
         war_num = int(row['war_num'])
         for x in clean(row['trans_from']):
             load_link(x, war_num)
         for x in clean(row['trans_to']):
             load_link(war_num, x)
Esempio n. 3
0
 def load_file(src):
     reader = csv2.DictReader(src, encoding='latin-1')
     reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
     for row in reader:
         war_num = int(row['war_num'])
         for x in clean(row['trans_from']):
             load_link(x, war_num)
         for x in clean(row['trans_to']):
             load_link(war_num, x)
Esempio n. 4
0
def load_cow_majors(src):
    """ Load data into cow_majors """
    session = model.SESSION()
    reader = csv2.DictReader(src)
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    cnt = collections.Counter()
    for row in reader:
        ccode = row['ccode']
        cnt[row['ccode']] +=1
        st_date = utils.row_ymd(row, 'st_year', 'st_month', 'st_day')
        end_date = utils.row_ymd(row, 'end_year', 'end_month', 'end_day')
        session.add(model.CowMajor(ccode = ccode,
                                    interval = cnt[ccode],
                                    st_date = st_date,
                                    end_date = end_date))
    session.commit()
Esempio n. 5
0
def load_cow_majors(src):
    """ Load data into cow_majors """
    session = model.SESSION()
    reader = csv2.DictReader(src)
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    cnt = collections.Counter()
    for row in reader:
        ccode = row['ccode']
        cnt[row['ccode']] += 1
        st_date = utils.row_ymd(row, 'st_year', 'st_month', 'st_day')
        end_date = utils.row_ymd(row, 'end_year', 'end_month', 'end_day')
        session.add(
            model.CowMajor(ccode=ccode,
                           interval=cnt[ccode],
                           st_date=st_date,
                           end_date=end_date))
    session.commit()
Esempio n. 6
0
def load_war3_partic(src):
    """ Load COW War Data v. 3, Participants """
    session = model.SESSION()
    def _int(x):
        try:
            return int(x)
        except TypeError:
            return None

    def _dates(row, n):
        if row['yr_beg%d' % n]:
            y = model.War3ParticDate()
            y.war_partic = row['war_partic']
            y.spell_no = n
            date_beg = utils.daterng(_int(row['yr_beg%d' % n]),
                                     _int(row['mon_beg%d' % n]),
                                     _int(row['day_beg%d' % n]))
            y.date_beg_min, y.date_beg_max = date_beg
            date_end = utils.daterng(_int(row['yr_end%d' % n]),
                                     _int(row['mon_end%d' % n]),
                                     _int(row['day_end%d' % n]))
            y.date_end_min, y.date_end_max = date_end
            session.add(y)

    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    war_cols = [x.name for x in model.War3Partic.__table__.c]
    war_date_cols = [x.name for x in model.War3ParticDate.__table__.c]
    cnt = collections.Counter()
    for row in reader:
        ## Account for multiple country-war participations
        key = (row['war_no'], row['state_num'])
        cnt[key] += 1
        row['partic_no'] = cnt[key]
        row['war_partic'] = war_partic_pkey(row['war_no'],
                                            row['state_num'],
                                            row['partic_no'])
        ## replace missing values
        for k,v in row.iteritems():
            row[k] = utils.replmiss(v, lambda x: x in ("-999", "-888"))
        session.add(model.War3Partic(**utils.subset(row, war_cols)))
        ## Dates
        for i in (1, 2):
             _dates(row, i)
    session.commit()
Esempio n. 7
0
def load_war3_partic(src):
    """ Load COW War Data v. 3, Participants """
    session = model.SESSION()

    def _int(x):
        try:
            return int(x)
        except TypeError:
            return None

    def _dates(row, n):
        if row['yr_beg%d' % n]:
            y = model.War3ParticDate()
            y.war_partic = row['war_partic']
            y.spell_no = n
            date_beg = utils.daterng(_int(row['yr_beg%d' % n]),
                                     _int(row['mon_beg%d' % n]),
                                     _int(row['day_beg%d' % n]))
            y.date_beg_min, y.date_beg_max = date_beg
            date_end = utils.daterng(_int(row['yr_end%d' % n]),
                                     _int(row['mon_end%d' % n]),
                                     _int(row['day_end%d' % n]))
            y.date_end_min, y.date_end_max = date_end
            session.add(y)

    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    war_cols = [x.name for x in model.War3Partic.__table__.c]
    war_date_cols = [x.name for x in model.War3ParticDate.__table__.c]
    cnt = collections.Counter()
    for row in reader:
        ## Account for multiple country-war participations
        key = (row['war_no'], row['state_num'])
        cnt[key] += 1
        row['partic_no'] = cnt[key]
        row['war_partic'] = war_partic_pkey(row['war_no'], row['state_num'],
                                            row['partic_no'])
        ## replace missing values
        for k, v in row.iteritems():
            row[k] = utils.replmiss(v, lambda x: x in ("-999", "-888"))
        session.add(model.War3Partic(**utils.subset(row, war_cols)))
        ## Dates
        for i in (1, 2):
            _dates(row, i)
    session.commit()
Esempio n. 8
0
def load_mid_links(src):
    """ Load tables mid_link_mid and link_mid_war """
    session = model.SESSION()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        disp_num = row['disp_num']
        for k in ('link%d' % i for i in range(1, 4)):
            link = row[k]
            if link == '0' or link is None:
                continue
            elif link[-1] == 'W':
                session.add(
                    model.MidLinkWar(disp_num=disp_num, war_num=link[:-1]))
            else:
                session.add(
                    model.MidLinkMid(disp_num_1=disp_num, disp_num_2=link))
    session.commit()
Esempio n. 9
0
File: mid.py Progetto: jrnold/irdata
def load_mid_links(src):
    """ Load tables mid_link_mid and link_mid_war """
    session = model.SESSION()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        disp_num = row['disp_num']
        for k in ('link%d' % i for i in range(1, 4)):
            link = row[k]
            if link == '0' or link is None:
                continue
            elif link[-1] == 'W':
                session.add(model.MidLinkWar(disp_num = disp_num,
                                             war_num = link[:-1]))
            else:
                session.add(model.MidLinkMid(disp_num_1 = disp_num,
                                             disp_num_2 = link))
    session.commit()
Esempio n. 10
0
File: mid.py Progetto: jrnold/irdata
def load_midi(src):
    session = model.SESSION()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    cols = [c.name for c in model.MidI.__table__.columns]
    int_cols = [c.name for c in model.MidI.__table__.columns
                if isinstance(c.type, sa.types.Integer)]
    for row in reader:
        start_date = utils.daterng(*(_int(row[k]) for k in
                                     ('st_year', 'st_mon', 'st_day')))
        row['st_date_min'], row['st_date_min'] = start_date
        end_date = utils.daterng(*(_int(row[k]) for k in
                                   ('end_year', 'end_mon', 'end_day')))
        row['end_date_min'], row['end_date_min'] = end_date
        ## set -9 to NULL
        for k in int_cols:
            row[k] = _int(row[k])
        session.add(model.MidI(**utils.subset(row, cols)))
    session.commit()
Esempio n. 11
0
def load_war3(src):
    """ Load COW War Data v. 3 """

    session = model.SESSION()

    def _int(x):
        try:
            return int(x)
        except TypeError:
            return None

    def _dates(row, n):
        if row['yr_beg%d' % n]:
            y = model.War3Date()
            y.war_no = row['war_no']
            y.spell_no = n
            date_beg = utils.daterng(_int(row['yr_beg%d' % n]),
                                     _int(row['mon_beg%d' % n]),
                                     _int(row['day_beg%d' % n]))
            y.date_beg_min, y.date_beg_max = date_beg
            date_end = utils.daterng(_int(row['yr_end%d' % n]),
                                     _int(row['mon_end%d' % n]),
                                     _int(row['day_end%d' % n]))
            y.date_end_min, y.date_end_max = date_end
            session.add(y)

    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]

    war_cols = [x.name for x in model.War3.__table__.c]
    war_date_cols = [x.name for x in model.War3Date.__table__.c]
    for row in reader:
        for k,v in row.iteritems():
            row[k] = utils.replmiss(v, lambda x: x in ("-999", "-888"))
        ## Inter-state war does not have a war_type
        if 'war_type' not in row.keys():
            row['war_type'] = 1
        row['oceania'] = row['oceania'] if row['oceania'] else False
        session.add(model.War3(**utils.subset(row, war_cols)))
        ## Dates
        for i in (1, 2):
            _dates(row, i)
    session.commit()
Esempio n. 12
0
def load_war3(src):
    """ Load COW War Data v. 3 """

    session = model.SESSION()

    def _int(x):
        try:
            return int(x)
        except TypeError:
            return None

    def _dates(row, n):
        if row['yr_beg%d' % n]:
            y = model.War3Date()
            y.war_no = row['war_no']
            y.spell_no = n
            date_beg = utils.daterng(_int(row['yr_beg%d' % n]),
                                     _int(row['mon_beg%d' % n]),
                                     _int(row['day_beg%d' % n]))
            y.date_beg_min, y.date_beg_max = date_beg
            date_end = utils.daterng(_int(row['yr_end%d' % n]),
                                     _int(row['mon_end%d' % n]),
                                     _int(row['day_end%d' % n]))
            y.date_end_min, y.date_end_max = date_end
            session.add(y)

    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]

    war_cols = [x.name for x in model.War3.__table__.c]
    war_date_cols = [x.name for x in model.War3Date.__table__.c]
    for row in reader:
        for k, v in row.iteritems():
            row[k] = utils.replmiss(v, lambda x: x in ("-999", "-888"))
        ## Inter-state war does not have a war_type
        if 'war_type' not in row.keys():
            row['war_type'] = 1
        row['oceania'] = row['oceania'] if row['oceania'] else False
        session.add(model.War3(**utils.subset(row, war_cols)))
        ## Dates
        for i in (1, 2):
            _dates(row, i)
    session.commit()
Esempio n. 13
0
def load_cow_states(src):
    """ Load data into cow_statelist and cow_system_membership """
    session = model.SESSION()
    reader = csv2.DictReader(src)
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    cnt = collections.Counter()
    for row in reader:
        ccode = row['ccode']
        cnt[row['ccode']] +=1
        if cnt[ccode] == 1:
            session.add(model.CowState(ccode = ccode,
                                       state_abb = row['state_abb'],
                                       state_nme = row['state_nme']))
        st_date = utils.row_ymd(row, 'st_year', 'st_month', 'st_day')
        end_date = utils.row_ymd(row, 'end_year', 'end_month', 'end_day')
        session.add(model.CowSysMembership(ccode = ccode,
                                           interval = cnt[ccode],
                                           st_date = st_date,
                                           end_date = end_date))
    session.commit()
Esempio n. 14
0
def load_cow_states(src):
    """ Load data into cow_statelist and cow_system_membership """
    session = model.SESSION()
    reader = csv2.DictReader(src)
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    cnt = collections.Counter()
    for row in reader:
        ccode = row['ccode']
        cnt[row['ccode']] += 1
        if cnt[ccode] == 1:
            session.add(
                model.CowState(ccode=ccode,
                               state_abb=row['state_abb'],
                               state_nme=row['state_nme']))
        st_date = utils.row_ymd(row, 'st_year', 'st_month', 'st_day')
        end_date = utils.row_ymd(row, 'end_year', 'end_month', 'end_day')
        session.add(
            model.CowSysMembership(ccode=ccode,
                                   interval=cnt[ccode],
                                   st_date=st_date,
                                   end_date=end_date))
    session.commit()
Esempio n. 15
0
def load_war4_nonstate(src):
    def _int(x):
        try:
            y = int(re.sub(',', '', x))
            return y if y > 0 else None
        except TypeError:
            return None

    def _side(x):
        return x if x != "-8" else  None

    def add_belligerent(session, name):
        ccode = None
        if name != "-8":
            belligerent = belligerent_key(ccode, name)
            q = session.query(model.War4Belligerent).\
                filter(model.War4Belligerent.belligerent == belligerent)
            if q.count() == 0:
                obj = model.War4Belligerent(
                    belligerent = belligerent,
                    belligerent_name = name,
                    ccode = ccode)
                session.add(obj)
            session.flush()
            
    def partic(row, side, name):
        y = model.War4Partic()
        war_side = war_side_pkey(_int(row['war_num']), (side == "b") + 1)
        belligerent = belligerent_key(None, name)
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.war_side = war_side
        y.belligerent = belligerent
        for k, v in WHERE_FOUGHT[_int(row['where_fought'])].iteritems():
            setattr(y, k, v)
        outcome = _int(row['outcome'])
        if side:
            if outcome == 2: outcome = 1
            elif outcome == 1: outcome = 2
        y.outcome = outcome
        y.initiator = (row['initiator'] == side.upper())
        return y

    def add_partic_dates(row, name, side):
        y = model.War4ParticDate()
        war_side = war_side_pkey(_int(row['war_num']),
                                 (side == "b") + 1)
        belligerent = belligerent_key(None, name)
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.partic_num = 1
        start_date = utils.daterng(_int(row['start_year']),
                                   _int(row['start_month']),
                                   _int(row['start_day']))
        y.start_date_min, y.start_date_max = start_date
        if row['end_year'] == "-7":
            y.end_date_min = y.end_date_max = model.War4.ONGOING_DATE
            y.ongoing = True
        else:
            end_date = utils.daterng(_int(row['end_year']),
                                     _int(row['end_month']),
                                     _int(row['end_day']))
            y.end_date_min, y.end_date_max = end_date
            y.ongoing = False
        session.add(y)

    session = model.SESSION()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        war_num = row['war_num']
        ## Add war
        session.add(model.War4(war_num = war_num,
                               war_name = row['war_name'],
                               war_type = int(row['war_type']),
                               bat_deaths = _int(row['total_combat_deaths'])))
        for side in ('a', 'b'):
            side_bool = (side == 'b') + 1
            session.add(model.War4Side(war_side = war_side_pkey(war_num, side_bool),
                                       side = side_bool,
                                       war_num = war_num,
                                       bat_death = _int(row['side_%sdeaths' % side])))
        session.flush()
        for i in (1, 2):
            name = row['side_a%d' % i]
            if name != '-8':
                add_belligerent(session, name)
                session.add(partic(row, 'a', name))
                add_partic_dates(row, name, 'a')
        for i in range(1, 6):
            name = row['side_b%d' % i]
            if name != '-8':
                add_belligerent(session, name)
                session.add(partic(row, 'b', name))
                add_partic_dates(row, name, 'b')
        session.flush()
    session.commit()
Esempio n. 16
0
def load_war4_intra(src):
    """ Add Intra-state war data to war4_* tables

    updates tables cow_war4, cow_belligerents, cow_war4_participation, cow_war4_partic_dates
    """
    session = model.SESSION()

    def _int(x):
        try:
            y = int(re.sub(',', '', x))
            return y if y > 0 else None
        except TypeError:
            return None

    def _side(x):
        return x if x != "-8" else  None

    def add_belligerent(session, name, ccode):
        if name != "-8":
            belligerent = belligerent_key(ccode, name)
            q = session.query(model.War4Belligerent).\
                filter(model.War4Belligerent.belligerent == belligerent)
            if q.count() == 0:
                obj = model.War4Belligerent(
                    belligerent = belligerent,
                    belligerent_name = name,
                    ccode = _int(ccode))
                session.add(obj)
            session.flush()
            
    def partic(row, belligerent, side):
        y = model.War4Partic()
        war_side = war_side_pkey(_int(row['war_num']),
                                 (side == 'b') + 1)
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.belligerent = belligerent
        y.war_side = war_side
        for k, v in WHERE_FOUGHT[_int(row['where_fought'])].iteritems():
            setattr(y, k, v)
        ## outcomes given in Side A / Side B rather than winner/loser
        ## per participant
        outcome = _int(row['outcome'])
        if side == 2:
            if outcome == 2: outcome = 1
            elif outcome == 1: outcome = 2
        y.outcome = outcome
        y.bat_death = _int(row['side_%sdeaths' % side])
        y.initiator = (row['initiator'] == y.belligerent)
        return y

    def add_partic_dates(row, belligerent, side, n):
        if row['start_year%d' % n] != '-8':
            y = model.War4ParticDate()
            war_side = war_side_pkey(_int(row['war_num']),
                                     (side == 'b') + 1)
            y.war_partic = war_partic_pkey(war_side,
                                           belligerent)
            y.belligerent = belligerent
            y.war_side = war_side
            y.partic_num = n
            start_date = utils.daterng(_int(row['start_year%d' % n]),
                                       _int(row['start_month%d' % n]),
                                       _int(row['start_day%d' % n]))
            y.start_date_min, y.start_date_max = start_date
            if row['end_year%d' % n] == "-7":
                y.end_date_min = y.end_date_max = model.War4.ONGOING_DATE
                y.ongoing = True
            else:
                end_date = utils.daterng(_int(row['end_year%d' % n]),
                                         _int(row['end_month%d' % n]),
                                         _int(row['end_day%d' % n]))
                y.end_date_min, y.end_date_max = end_date
                y.ongoing = False
            session.add(y)
    
    cnt = collections.Counter()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        war_num = row['war_num']
        cnt[war_num] += 1
        ## Add war
        if cnt[war_num] == 1:
            session.add(model.War4(war_num = war_num,
                                      war_name = row['war_name'],
                                      war_type = int(row['war_type']),
                                      intnl = row['intnl'] == '1'))
            for side in (1, 2):
                session.add(model.War4Side(war_side = war_side_pkey(war_num, side),
                                           side=side, war_num=row['war_num']))
            session.flush()
        if _side(row['side_a']):
            add_belligerent(session, row['side_a'], row['ccode_a'])
            belligerent = belligerent_key(row['ccode_a'], row['side_a'])
            session.add(partic(row, belligerent, 'a'))
            for i in (1, 2):
                add_partic_dates(row, belligerent, 'a', i)
        if _side(row['side_b']):
            add_belligerent(session, row['side_b'], row['ccode_b'])
            belligerent = belligerent_key(row['ccode_b'], row['side_b'])
            session.add(partic(row, belligerent, 'b'))
            for i in (1, 2):
                add_partic_dates(row, belligerent, 'b', i)
        session.flush()
    session.commit()
Esempio n. 17
0
def load_war4_inter(src):
    """ Add Inter-state war data to war4_* tables
    
    updates tables cow_war4, cow_belligerents, cow_war4_participation, cow_war4_partic_dates
    """
    session = model.SESSION()

    def _int(x):
        y = int(x)
        return y if y >= 0 else None

    def partic(row):
        y = model.War4Partic()
        belligerent = belligerent_key(row['ccode'], row['state_name'])
        war_side = war_side_pkey(int(row['war_num']),
                                 int(row['side']))
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.war_side = war_side
        y.belligerent = belligerent
        for k, v in WHERE_FOUGHT[_int(row['where_fought'])].iteritems():
            setattr(y, k, v)
        y.outcome = row['outcome']
        y.bat_death = _int(row['bat_death'])
        y.initiator = (int(row['initiator']) == 1)
        return y

    def add_partic_dates(row, n):
        if row['start_year%d' % n] != '-8':
            y = model.War4ParticDate()
            war_side = war_side_pkey(int(row['war_num']),
                                     int(row['side']))
            belligerent = belligerent_key(row['ccode'], row['state_name'])
            y.war_partic = war_partic_pkey(war_side, belligerent)
            y.partic_num = n
            start_date = utils.daterng(_int(row['start_year%d' % n]),
                                       _int(row['start_month%d' % n]),
                                       _int(row['start_day%d' % n]))
            y.start_date_min, y.start_date_max = start_date
            if row['end_year%d' % n] == "-7":
                y.end_date_min = y.end_date_max = model.War4.ONGOING_DATE
                y.ongoing = True
            else:
                end_date = utils.daterng(_int(row['end_year%d' % n]),
                                         _int(row['end_month%d' % n]),
                                         _int(row['end_day%d' % n]))
                y.end_date_min, y.end_date_max = end_date
                y.ongoing = False
            session.add(y)
        
    cols = ("war_num", "war_name", "war_type")
    cnt = collections.Counter()
    cnt_bellig = collections.Counter()
    reader = csv2.DictReader(src)
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        war_num = row['war_num']
        belligerent = belligerent_key(row['ccode'], row['state_name'])
        cnt[war_num] += 1
        cnt_bellig[belligerent] +=1 
        if cnt[war_num] == 1:
            session.add(model.War4(intnl=True, **utils.subset(row, cols)))
            for side in (1, 2):
                session.add(model.War4Side(war_side = war_side_pkey(side=side, war_num=war_num),
                                           side=side, war_num=war_num))
            session.flush()
        if cnt_bellig[belligerent] == 1:
            session.add(model.War4Belligerent(belligerent = belligerent,
                                              belligerent_name = row['state_name'],
                                              ccode = row['ccode']))
            session.flush()
        session.add(partic(row))
        for i in (1, 2):
            add_partic_dates(row, i)
    session.commit()
Esempio n. 18
0
def load_war4_inter(src):
    """ Add Inter-state war data to war4_* tables
    
    updates tables cow_war4, cow_belligerents, cow_war4_participation, cow_war4_partic_dates
    """
    session = model.SESSION()

    def _int(x):
        y = int(x)
        return y if y >= 0 else None

    def partic(row):
        y = model.War4Partic()
        belligerent = belligerent_key(row['ccode'], row['state_name'])
        war_side = war_side_pkey(int(row['war_num']), int(row['side']))
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.war_side = war_side
        y.belligerent = belligerent
        for k, v in WHERE_FOUGHT[_int(row['where_fought'])].iteritems():
            setattr(y, k, v)
        y.outcome = row['outcome']
        y.bat_death = _int(row['bat_death'])
        y.initiator = (int(row['initiator']) == 1)
        return y

    def add_partic_dates(row, n):
        if row['start_year%d' % n] != '-8':
            y = model.War4ParticDate()
            war_side = war_side_pkey(int(row['war_num']), int(row['side']))
            belligerent = belligerent_key(row['ccode'], row['state_name'])
            y.war_partic = war_partic_pkey(war_side, belligerent)
            y.partic_num = n
            start_date = utils.daterng(_int(row['start_year%d' % n]),
                                       _int(row['start_month%d' % n]),
                                       _int(row['start_day%d' % n]))
            y.start_date_min, y.start_date_max = start_date
            if row['end_year%d' % n] == "-7":
                y.end_date_min = y.end_date_max = model.War4.ONGOING_DATE
                y.ongoing = True
            else:
                end_date = utils.daterng(_int(row['end_year%d' % n]),
                                         _int(row['end_month%d' % n]),
                                         _int(row['end_day%d' % n]))
                y.end_date_min, y.end_date_max = end_date
                y.ongoing = False
            session.add(y)

    cols = ("war_num", "war_name", "war_type")
    cnt = collections.Counter()
    cnt_bellig = collections.Counter()
    reader = csv2.DictReader(src)
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        war_num = row['war_num']
        belligerent = belligerent_key(row['ccode'], row['state_name'])
        cnt[war_num] += 1
        cnt_bellig[belligerent] += 1
        if cnt[war_num] == 1:
            session.add(model.War4(intnl=True, **utils.subset(row, cols)))
            for side in (1, 2):
                session.add(
                    model.War4Side(war_side=war_side_pkey(side=side,
                                                          war_num=war_num),
                                   side=side,
                                   war_num=war_num))
            session.flush()
        if cnt_bellig[belligerent] == 1:
            session.add(
                model.War4Belligerent(belligerent=belligerent,
                                      belligerent_name=row['state_name'],
                                      ccode=row['ccode']))
            session.flush()
        session.add(partic(row))
        for i in (1, 2):
            add_partic_dates(row, i)
    session.commit()
Esempio n. 19
0
def load_war4_intra(src):
    """ Add Intra-state war data to war4_* tables

    updates tables cow_war4, cow_belligerents, cow_war4_participation, cow_war4_partic_dates
    """
    session = model.SESSION()

    def _int(x):
        try:
            y = int(re.sub(',', '', x))
            return y if y > 0 else None
        except TypeError:
            return None

    def _side(x):
        return x if x != "-8" else None

    def add_belligerent(session, name, ccode):
        if name != "-8":
            belligerent = belligerent_key(ccode, name)
            q = session.query(model.War4Belligerent).\
                filter(model.War4Belligerent.belligerent == belligerent)
            if q.count() == 0:
                obj = model.War4Belligerent(belligerent=belligerent,
                                            belligerent_name=name,
                                            ccode=_int(ccode))
                session.add(obj)
            session.flush()

    def partic(row, belligerent, side):
        y = model.War4Partic()
        war_side = war_side_pkey(_int(row['war_num']), (side == 'b') + 1)
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.belligerent = belligerent
        y.war_side = war_side
        for k, v in WHERE_FOUGHT[_int(row['where_fought'])].iteritems():
            setattr(y, k, v)
        ## outcomes given in Side A / Side B rather than winner/loser
        ## per participant
        outcome = _int(row['outcome'])
        if side == 2:
            if outcome == 2: outcome = 1
            elif outcome == 1: outcome = 2
        y.outcome = outcome
        y.bat_death = _int(row['side_%sdeaths' % side])
        y.initiator = (row['initiator'] == y.belligerent)
        return y

    def add_partic_dates(row, belligerent, side, n):
        if row['start_year%d' % n] != '-8':
            y = model.War4ParticDate()
            war_side = war_side_pkey(_int(row['war_num']), (side == 'b') + 1)
            y.war_partic = war_partic_pkey(war_side, belligerent)
            y.belligerent = belligerent
            y.war_side = war_side
            y.partic_num = n
            start_date = utils.daterng(_int(row['start_year%d' % n]),
                                       _int(row['start_month%d' % n]),
                                       _int(row['start_day%d' % n]))
            y.start_date_min, y.start_date_max = start_date
            if row['end_year%d' % n] == "-7":
                y.end_date_min = y.end_date_max = model.War4.ONGOING_DATE
                y.ongoing = True
            else:
                end_date = utils.daterng(_int(row['end_year%d' % n]),
                                         _int(row['end_month%d' % n]),
                                         _int(row['end_day%d' % n]))
                y.end_date_min, y.end_date_max = end_date
                y.ongoing = False
            session.add(y)

    cnt = collections.Counter()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        war_num = row['war_num']
        cnt[war_num] += 1
        ## Add war
        if cnt[war_num] == 1:
            session.add(
                model.War4(war_num=war_num,
                           war_name=row['war_name'],
                           war_type=int(row['war_type']),
                           intnl=row['intnl'] == '1'))
            for side in (1, 2):
                session.add(
                    model.War4Side(war_side=war_side_pkey(war_num, side),
                                   side=side,
                                   war_num=row['war_num']))
            session.flush()
        if _side(row['side_a']):
            add_belligerent(session, row['side_a'], row['ccode_a'])
            belligerent = belligerent_key(row['ccode_a'], row['side_a'])
            session.add(partic(row, belligerent, 'a'))
            for i in (1, 2):
                add_partic_dates(row, belligerent, 'a', i)
        if _side(row['side_b']):
            add_belligerent(session, row['side_b'], row['ccode_b'])
            belligerent = belligerent_key(row['ccode_b'], row['side_b'])
            session.add(partic(row, belligerent, 'b'))
            for i in (1, 2):
                add_partic_dates(row, belligerent, 'b', i)
        session.flush()
    session.commit()
Esempio n. 20
0
def load_war4_nonstate(src):
    def _int(x):
        try:
            y = int(re.sub(',', '', x))
            return y if y > 0 else None
        except TypeError:
            return None

    def _side(x):
        return x if x != "-8" else None

    def add_belligerent(session, name):
        ccode = None
        if name != "-8":
            belligerent = belligerent_key(ccode, name)
            q = session.query(model.War4Belligerent).\
                filter(model.War4Belligerent.belligerent == belligerent)
            if q.count() == 0:
                obj = model.War4Belligerent(belligerent=belligerent,
                                            belligerent_name=name,
                                            ccode=ccode)
                session.add(obj)
            session.flush()

    def partic(row, side, name):
        y = model.War4Partic()
        war_side = war_side_pkey(_int(row['war_num']), (side == "b") + 1)
        belligerent = belligerent_key(None, name)
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.war_side = war_side
        y.belligerent = belligerent
        for k, v in WHERE_FOUGHT[_int(row['where_fought'])].iteritems():
            setattr(y, k, v)
        outcome = _int(row['outcome'])
        if side:
            if outcome == 2: outcome = 1
            elif outcome == 1: outcome = 2
        y.outcome = outcome
        y.initiator = (row['initiator'] == side.upper())
        return y

    def add_partic_dates(row, name, side):
        y = model.War4ParticDate()
        war_side = war_side_pkey(_int(row['war_num']), (side == "b") + 1)
        belligerent = belligerent_key(None, name)
        y.war_partic = war_partic_pkey(war_side, belligerent)
        y.partic_num = 1
        start_date = utils.daterng(_int(row['start_year']),
                                   _int(row['start_month']),
                                   _int(row['start_day']))
        y.start_date_min, y.start_date_max = start_date
        if row['end_year'] == "-7":
            y.end_date_min = y.end_date_max = model.War4.ONGOING_DATE
            y.ongoing = True
        else:
            end_date = utils.daterng(_int(row['end_year']),
                                     _int(row['end_month']),
                                     _int(row['end_day']))
            y.end_date_min, y.end_date_max = end_date
            y.ongoing = False
        session.add(y)

    session = model.SESSION()
    reader = csv2.DictReader(src, encoding='latin1')
    reader.fieldnames = [utils.camel2under(x) for x in reader.fieldnames]
    for row in reader:
        war_num = row['war_num']
        ## Add war
        session.add(
            model.War4(war_num=war_num,
                       war_name=row['war_name'],
                       war_type=int(row['war_type']),
                       bat_deaths=_int(row['total_combat_deaths'])))
        for side in ('a', 'b'):
            side_bool = (side == 'b') + 1
            session.add(
                model.War4Side(war_side=war_side_pkey(war_num, side_bool),
                               side=side_bool,
                               war_num=war_num,
                               bat_death=_int(row['side_%sdeaths' % side])))
        session.flush()
        for i in (1, 2):
            name = row['side_a%d' % i]
            if name != '-8':
                add_belligerent(session, name)
                session.add(partic(row, 'a', name))
                add_partic_dates(row, name, 'a')
        for i in range(1, 6):
            name = row['side_b%d' % i]
            if name != '-8':
                add_belligerent(session, name)
                session.add(partic(row, 'b', name))
                add_partic_dates(row, name, 'b')
        session.flush()
    session.commit()