def test_get_datetime():
s = '\n'
s += 'get_datetime() = ' + str(util.get_datetime()) + '\n'
s += 'get_datetime(1546400096.987654) = ' + str(
util.get_datetime(1546400096.987654)) + '\n'
s += 'get_datetime(1546400096) = ' + str(
util.get_datetime(1546400096)) + '\n'
s += 'get_datetime(\'20190102123456987654\', \'%Y%m%d%H%M%S%f\') = ' + str(
util.get_datetime('20190102123456987654', '%Y%m%d%H%M%S%f')) + '\n'
s += 'get_datetime(\'2019-02-21 12:34:56.789\', \'%Y-%m-%d %H:%M:%S.%f\') = ' + str(
util.get_datetime('2019-02-21 12:34:56.789',
'%Y-%m-%d %H:%M:%S.%f')) + '\n'
s += 'get_datetime(\'1234\', \'%H%M\') = ' + str(
util.get_datetime('1234', '%H%M')) + '\n'
obj = util.get_datetime('2019-01-02 12:34:56.987654')
s += str(obj) + '\n'
obj = util.get_datetime('20190102123456987654', '%Y%m%d%H%M%S%f')
s += str(obj) + '\n'
obj = util.get_datetime('20190102123456', '%Y%m%d%H%M%S')
s += str(obj) + '\n'
obj = util.get_datetime('201901021234', '%Y%m%d%H%M')
s += str(obj) + '\n'
return s
def get_season_dates(year):
"""Return list of season start and playoff start dates for NBA season ending in year"""
url = f"https://en.wikipedia.org/wiki/{year-1}-{year % 100}_NBA_season"
if year % 100 < 10:
url = f"https://en.wikipedia.org/wiki/{year-1}-0{year % 100}_NBA_season"
response = requests.get(url)
if response.status_code == 200:
season_soup = bs4.BeautifulSoup(response.text, "html.parser")
dates = season_soup("table", class_="infobox")[0].select("tbody tr")[3]
date_string = None
for i, d in enumerate(dates):
if i != 0:
date_string = d.get_text(separator="\n")
date_list = date_string.split("\n")
date_list = [
date.replace("\xa0", " ").replace(" (Playoffs)", "").strip()
for date in date_list
]
season_start = date_list[0].split(" – ")[0].strip()
if year == 1999:
season_start += " 1999"
playoff_start = date_list[1].split("–")[0].strip()
if playoff_start[-4:] != str(year):
playoff_start += f" {year}"
season_start = get_datetime(season_start)
playoff_start = get_datetime(playoff_start)
else:
response.raise_for_status()
return [season_start, playoff_start]
def get_season_info():
with open("data/season_info.csv", mode="w") as file:
writer = csv.writer(file,
delimiter="|",
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
for year in range(1947, 2020):
i = year - 1946
season_start, playoff_start = get_season_dates(year)
season_info = [i, 1, year, season_start, playoff_start]
writer.writerow(season_info)
start_dates = [
"October 13, 1967", "October 18, 1968", "October 17, 1969",
"October 14, 1970", "October 13, 1971", "October 12, 1972",
"October 10, 1973", "October 18, 1974", "October 24, 1975"
]
playoff_dates = [
"March 23, 1968", "April 5, 1969", "April 17, 1970",
"April 1, 1971", "March 31, 1972", "March 30, 1973",
"March 29, 1974", "April 4, 1975", "April 8, 1976"
]
i += 1
for year in range(1968, 1977):
j = year - 1968
season_info = [
i, 2, year,
get_datetime(start_dates[j]),
get_datetime(playoff_dates[j])
]
writer.writerow(season_info)
i += 1
def main_scan(stream_info):
try:
if "SUB_PATH" in stream_info:
os.environ["PATH"] = stream_info[
'SUB_PATH'] + os.pathsep + os.environ["PATH"]
current_path = sys.path[0]
tool_bin = os.path.join(current_path, 'tools')
content = util.base64toencode(
util.str_to_bytes(json.dumps(stream_info)))
ex_py_script = ''
if 'TOOL_TYPE' in stream_info:
ex_py_script = stream_info['TOOL_TYPE'] + '_ext.py'
command = "python " + tool_bin + '/' + ex_py_script + " " + content
print(util.get_datetime() + " start scan " + stream_info['TOOL_TYPE'] +
"...")
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
start_new_session=True)
pid_config.add_pid(str(proc.pid), stream_info["PID_FILE"])
for line in proc.stdout:
print(util.get_datetime() + " " + str(line))
print(util.get_datetime() + " end scan " + stream_info['TOOL_TYPE'])
except Exception as e:
raise Exception(e)
finally:
proc.terminate()
proc.wait()
def add_timestamp(df, dt_field, ts_field):
df = df.dropna(subset=[dt_field])
df[ts_field] = map(lambda dt: time.mktime(
get_datetime(dt).timetuple()
),
df[dt_field])
return df
def test_get_datetime():
data = [
994832962, 966810600000L, '2004-04-28 00:00:00.000',
'2004-04-28 00:00:00',
datetime.fromtimestamp(994832962)
]
for d in data:
assert_true(isinstance(get_datetime(d), datetime))
assert_raises(TypeError, get_datetime, dict())
assert_raises(ValueError, get_datetime, 'bad formatsadfasfd')
def get_page_results(self, html):
results = []
soup = BeautifulSoup(html, 'html.parser')
if len(soup.select('.card-no-result')) > 0:
return None
if len(soup.select('div[action-type="feed_list_item"]')) > 0:
for item in soup.select('div[action-type="feed_list_item"]'):
blog = {'博主昵称': item.select('.name')[0].get('nick-name'),
'博主主页': 'https:' + item.select('.name')[0].get('href')}
if len(item.select('p.txt')) > 1:
blog['微博内容'] = item.select('p.txt')[1].get_text().strip()
else:
blog['微博内容'] = item.select('p.txt')[0].get_text().strip()
blog['发布时间'] = util.get_datetime(
item.select('div[class="content"] p[class="from"] a')[0].get_text())
blog['微博地址'] = 'https:' + item.select('div[class="content"] p[class="from"] a')[0].get('href')
try:
blog['微博来源'] = item.select('div[class="content"] p[class="from"] a')[1].get_text()
except:
blog['微博来源'] = ''
try:
sd = item.select('.card-act ul li')
if sd is None:
continue
except:
continue
try:
blog['转发'] = 0 if sd[1].text.replace('转发', '').strip() == '' \
else int(sd[1].text.replace('转发', '').strip())
except:
blog['转发'] = 0
try:
blog['评论'] = 0 if sd[2].text.replace('评论', '').strip() == '' \
else int(sd[2].text.replace('评论', '').strip())
except:
blog['评论'] = 0
try:
blog['赞'] = 0 if sd[3].select('em')[0].get_text() == '' \
else int(sd[3].select('em')[0].get_text())
except:
blog['赞'] = 0
results.append(blog)
return results
def test_get_datetime():
data = [994832962,
966810600000L,
'2004-04-28 00:00:00.000',
'2004-04-28 00:00:00',
datetime.fromtimestamp(994832962)]
for d in data:
assert_true(
isinstance(get_datetime(d),
datetime)
)
assert_raises(TypeError, get_datetime, dict())
assert_raises(ValueError, get_datetime, 'bad formatsadfasfd')
def __init__(self, fileName, mode):
pwd = os.path.dirname(os.path.realpath(__file__))
log_dir = os.path.join(pwd, "..", "result", "log")
demo, whom = find_demo_name(sys.argv)
f = fileName
_mode = mode
if whom:
suffix = util.get_datetime()
f = u'.'.join([demo, suffix, "txt"])
if whom == C.log_caller.PSKETCH:
_mode = 'a' # append, not overwrite
path = os.path.join(log_dir, f)
super(PasketFileHandler, self).__init__(path, _mode)
def __init__(self, fileName, mode):
pwd = os.path.dirname(os.path.realpath(__file__))
log_dir = os.path.join(pwd, "..", "result", "log")
demo, whom = find_demo_name(sys.argv)
f = fileName
_mode = mode
if whom:
suffix = util.get_datetime()
f = u'.'.join([demo, suffix, "txt"])
if whom == C.log_caller.PSKETCH:
_mode = 'a' # append, not overwrite
path = os.path.join(log_dir, f)
super(JskFileHandler, self).__init__(path, _mode)
def test_get_timestamp():
s = ''
timestamp = util.get_timestamp()
s += str(timestamp) + '\n'
dt = util.get_datetime('20190102123456987654', '%Y%m%d%H%M%S%f')
timestamp = util.get_timestamp(dt)
s += str(timestamp) + '\n'
timestamp = util.get_timestamp('2019-01-02 12:34:56.789')
s += str(timestamp) + '\n'
timestamp = util.get_timestamp('197001020900', '%Y%m%d%H%M')
s += str(timestamp) + '\n'
return s
def route_add_answer(question_id):
question_detail_url = url_for('route_question_detail',
question_id=question_id)
if request.method == 'POST':
new_answer = request.form.to_dict()
new_answer['submission_time'] = util.get_datetime()
new_answer['site_user_id'] = session['user_id']
data_manager.add_new_answer(new_answer)
return redirect(question_detail_url)
question_data = data_manager.get_question_data(question_id)
answers = data_manager.get_answers_for_question(question_id)
add_answer_url = url_for('route_add_answer', question_id=question_id)
return render_template('add_answer.html',
question_id=question_id,
add_answer_url=add_answer_url,
question_data=question_data,
answers=answers)
def add_answer_comment(answer_id):
question_detail_url = url_for(
'route_question_detail',
question_id=data_manager.get_question_id_for_answer(
answer_id)['question_id'])
if request.method == 'POST':
new_comment = request.form.to_dict()
new_comment['submission_time'] = util.get_datetime()
new_comment.update({'site_user_id': session['user_id']})
data_manager.add_new_answer_comment(new_comment)
return redirect(question_detail_url)
answer_data = data_manager.get_answer_data(answer_id)
add_comment_url = url_for('add_answer_comment', answer_id=answer_id)
return render_template('add_comment_answer.html',
answer_id=answer_id,
question_detail_url=question_detail_url,
add_comment_url=add_comment_url,
answer_data=answer_data)
def clean_interactions(self, interactions, undirected=False,
convert_time=True):
"""Some cleaning. Functional
"""
new_interactions = []
if isinstance(interactions, pd.DataFrame):
iters = interactions.iterrows()
else:
iters = enumerate(interactions)
for row_n, i in iters:
if row_n % 5000 == 0:
logger.debug("cleaning: {} / {}".format(
row_n,
len(interactions))
)
i = copy.deepcopy(i)
if not undirected:
# remove duplicate recipients
i['recipient_ids'] = list(set(i['recipient_ids']))
else:
i['participant_ids'] = list(set(i['participant_ids']))
if 'timestamp' in i:
i['datetime'] = i['timestamp']
if convert_time:
# normalize datetime and timestamp
try:
i['datetime'] = get_datetime(i['datetime'])
except TypeError:
logger.warn(
'Error parsing datetime, {} of type {}'.format(
i['datetime'],
type(i['datetime'])
)
)
continue
new_interactions.append(i)
return new_interactions
def test():
s = ''
s += 'test_get_datetime() = ' + test_get_datetime() + '\n'
s += '\n'
s += 'test_get_datetime_str() = ' + test_get_datetime_str() + '\n'
s += '\n'
s += 'test_get_timestamp() = \n' + test_get_timestamp() + '\n'
s += '\n'
s += 'test_get_datetime_of_day() = ' + test_get_datetime_of_day() + '\n'
s += '\n'
s += 'test_get_timestamp_of_day() = \n' + test_get_timestamp_of_day(
) + '\n'
s += '\n'
s += 'test_get_last_day_of_month() = \n' + test_get_last_day_of_month(
) + '\n'
s += '\n'
s += 'test_is_leap_year() = \n' + test_is_leap_year() + '\n'
s += '\n'
s += 'datetime:\n'
dt = util.get_datetime()
s += 'timestamp() = ' + str(dt.timestamp()) + '\n'
s += 'year = ' + str(dt.year) + '\n'
s += 'month = ' + str(dt.month) + '\n'
s += 'day = ' + str(dt.day) + '\n'
s += 'hour = ' + str(dt.hour) + '\n'
s += 'minute = ' + str(dt.minute) + '\n'
s += 'second = ' + str(dt.second) + '\n'
s += 'microsecond = ' + str(dt.microsecond) + '\n'
s += 'tzinfo = ' + str(dt.tzinfo) + '\n'
s += '\n'
return s
def main(inn,out):
root = {}
rec_counts = {}
# == HEADER - WSO00R ==
if True:
ELEMENT = "WSO00R"
SEGMENT = "00"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
HEAD_ID = inn.get({'BOTSID': 'picking'}, {'BOTSID': 'header', 'message_id': None})
HEAD_DATE, HEAD_TIME = get_datetime(inn.get({'BOTSID': 'picking'}, {'BOTSID': 'header', 'date_msg': None}))
HEAD_TYPE = inn.get({'BOTSID': 'picking'}, {'BOTSID': 'header', 'type': None})
assert HEAD_TYPE == 'out', "Warehouse Shipping Orders must have type 'out', not '%s'" % (HEAD_TYPE,)
HEAD_TYPE = "940" # Warehouse Shipping Order
d = OrderedDict([
('Segment_Number', SEGMENT),
('Identification_Nbr', HEAD_ID or ''),
('Message_Date', HEAD_DATE or ''),
('Message_Time', HEAD_TIME or ''),
('Message_Type_Id', HEAD_TYPE),
])
root.setdefault(ELEMENT, []).append(d)
pinn = inn.getloop({'BOTSID': 'picking'}, {'BOTSID': 'pickings'})
for pick in pinn:
ORD_ID = pick.get({'BOTSID': 'pickings', 'id': None})
assert ORD_ID, "Order ID must be present"
ORD_ID = re.sub(r'[\\/_-]', r'', ORD_ID.upper())
out.ta_info['botskey'] = ORD_ID # Set the botskey to the last order ID
ORD_STATE = pick.get({'BOTSID': 'pickings', 'state': None})
# == Cancelled Orders - WSO05R ==
if ORD_STATE == 'delete':
ELEMENT = "WSO05R"
SEGMENT = "05"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Order_reference', ORD_ID),
])
root.setdefault(ELEMENT, []).append(d)
# == New Orders - WSO10R ==
elif ORD_STATE == 'new':
ELEMENT = "WSO10R"
SEGMENT = "10"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
INCOTERM = pick.get({'BOTSID': 'pickings', 'incoterm': None})
ORD_PRIO = pick.get({'BOTSID': 'pickings', 'prio': None})
ORD_REMARK = pick.get({'BOTSID': 'pickings', 'desc': None})
ORD_DELIVERY_DATE, dummy = get_datetime(pick.get({'BOTSID': 'pickings', 'date': None}) + ' 00:00:00.00000')
ord_root = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Type', 'O'), # Outgoing
('Indicator_Transport', 'Y'),
('Order_Reference', ORD_ID),
('Indicator_Labelling', 'Y'),
('Indicator_contact_cl', 'Y'),
('Terms_of_Delivery', INCOTERM),
('COD_amount', '0.0'),
('Currency_COD-amount', ''),
('Order_Category', 'N'), # N = New
('Order_priority', ORD_PRIO),
('Delivery_date', ORD_DELIVERY_DATE),
('Remark', ORD_REMARK), # FIXME: We add this last to validate against DSV's example messages, it should come just before priority
])
root.setdefault(ELEMENT, []).append(ord_root)
# == Order Remarks - WSO20R ==
if ORD_REMARK:
ELEMENT = "WSO20R"
SEGMENT = "20"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Remark', ORD_REMARK),
('Order_Remark_Qual', 'D'), # Delivery Instructions
])
ord_root.setdefault(ELEMENT, []).append(d)
# == Order Party - WSO30R ==
ELEMENT = "WSO30R"
SEGMENT = "30"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
PART_ID = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'id': None})
CLIENT_REF = pick.get({'BOTSID': 'pickings', 'client_order_ref': None})
PART_EMAIL = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'email': None})
party_root = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Party_Qual', 'CN'), # Consignee - Party receiving goods
('Party_External_Ref', PART_ID),
('Client_Ref_to_Order', CLIENT_REF),
])
ord_root.setdefault(ELEMENT, []).append(party_root)
# == Order Party Email - WSO31R ==
if PART_EMAIL:
ELEMENT = "WSO31R"
SEGMENT = "31"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Communication_Number_Qualifier', 'EA'), # Email
('Communication_Number', PART_EMAIL),
('Recipient_type', 0), # 0=To, 1=CC, 2=BCC
])
party_root.setdefault(ELEMENT, []).append(d)
# == Order Party Address - WSO32R ==
ELEMENT = "WSO32R"
SEGMENT = "32"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
PART_NAME = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'name': None}) or ''
PART_STREET1 = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'street1': None}) or ''
PART_STREET2 = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'street2': None}) or ''
PART_CITY = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'city': None}) or ''
PART_ZIP = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'zip': None}) or ''
PART_COUNTRY = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'country': None}) or ''
PART_STATE = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'state': None}) or ''
PART_PHONE = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'phone': None}) or ''
PART_FAX = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'fax': None}) or ''
PART_LANG = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'language': None}) or ''
PART_VAT = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'vat': None}) or ''
# Spill the first address line over to the second if too long, else truncate
if len(PART_STREET1) > 35 and not PART_STREET2:
space = PART_STREET1.rfind(' ', 0, 35)
if space:
OLD_STREET1 = PART_STREET1
PART_STREET1 = OLD_STREET1[:space]
PART_STREET2 = OLD_STREET1[space+1:]
PART_STREET1 = PART_STREET1[:35]
PART_STREET2 = PART_STREET2[:35]
d = OrderedDict([
('Segment_Number', SEGMENT),
('Company_Name_1', PART_NAME[:35]),
('Address_3', PART_STREET2),
('Postal_Code', PART_ZIP),
('Place_Name', PART_CITY[:35]),
('Country_Code', PART_COUNTRY),
('Language_code', get_dsv_lang(PART_LANG)),
('Telephone_Number', PART_PHONE),
('Telefax_Number', PART_FAX),
('VAT-number', PART_VAT),
('Address_2', PART_STREET1),
('Address_4', PART_STATE),
])
party_root.setdefault(ELEMENT, []).append(d)
plines = pick.getloop({'BOTSID': 'pickings'}, {'BOTSID': 'line'})
for pline in plines:
# == Order Line - WSO40R ==
ELEMENT = "WSO40R"
SEGMENT = "40"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
LINE_ID = pline.get({'BOTSID': 'line', 'id': None})
LINE_SEQ = "%03d" % (int(pline.get({'BOTSID': 'line', 'seq': None})),)
LINE_PRODUCT = pline.get({'BOTSID': 'line', 'product': None}).upper()
LINE_TYPE = '*FIRST'
LINE_QTY = pline.get({'BOTSID': 'line', 'product_qty': None})
LINE_DESC = pline.get({'BOTSID': 'line', 'desc': None})
LINE_VOLUME_NET = pline.get({'BOTSID': 'line', 'volume_net': None})
LINE_WEIGHT = pline.get({'BOTSID': 'line', 'weight': None})
LINE_WEIGHT_NET = pline.get({'BOTSID': 'line', 'weight_net': None})
LINE_PRICE_UNIT = pline.get({'BOTSID': 'line', 'price_unit_ex_vat': None}) or 0.0
LINE_CURRENCY = pline.get({'BOTSID': 'line', 'price_currency': None})
line_root = OrderedDict([
('Segment_Number', SEGMENT),
('Orderline_ID', LINE_SEQ),
('SKU_Reference', LINE_PRODUCT),
('Type_of_package', LINE_TYPE),
('Number_of_packages', LINE_QTY),
('Number_of_units_measure_unit_specifier', LINE_QTY),
('Number_of_units_measure_unit_specifier_per_type_of_package', 1), # FIXME: We only support units UoM currently
('Orderline_remarks', LINE_DESC),
('Net_volume_per_unit_measure_unit-specifier', LINE_VOLUME_NET),
])
ord_root.setdefault(ELEMENT, []).append(line_root)
# == Order Line Invoice - WSO41R ==
ELEMENT = "WSO41R"
SEGMENT = "41"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
assert (ord_root.get('Currency_COD-amount', LINE_CURRENCY) or LINE_CURRENCY) == LINE_CURRENCY, 'Multiple currencies per order is not supported'
ord_root.update({'Currency_COD-amount': LINE_CURRENCY})
try:
LINE_PRICE_UNIT = "%.2f" % (round(float(LINE_PRICE_UNIT), 2),)
except ValueError:
LINE_PRICE_UNIT = ""
d = OrderedDict([
('Segment_Number', SEGMENT),
('Goods_invoice_amount', LINE_PRICE_UNIT),
('Goods_inv_amoun_Qual', 'U'), # Unit price
])
line_root.setdefault(ELEMENT, []).append(d)
# Catch all
else:
raise NotImplementedError('Unable to handle order with state %s' % (ORD_STATE,))
# == FOOTER - WSO99R ==
rec_counts['99'] = len(rec_counts.keys())
for FOOT_REC_TYPE, FOOT_REC_COUNT in rec_counts.iteritems():
ELEMENT = "WSO99R"
SEGMENT = "99"
d = OrderedDict([
('Segment_Number', SEGMENT),
('RecordType', FOOT_REC_TYPE),
('Total_Nbrs_of_Rec', FOOT_REC_COUNT),
])
root.setdefault(ELEMENT, []).append(d)
root = OrderedDict(sorted(root.items(), key=lambda t: t[0]))
document = OrderedDict([('WSO', root)])
data = serialize_dsv_xml(document, SCHEMA)
out.root = data
return
def main(inn,out):
data = parse_dsv_xml(inn.root, SCHEMA)
data = simplify_dsv_xml(data)
for BODY in data['SR']:
lout = out.putloop({'BOTSID':'inventory'})
rec_counts = {}
# == HEADER - SR00R ==
for HEAD in BODY.get('SR00R', []):
rec_counts['00'] = rec_counts.setdefault('00', 0) + 1
if rec_counts['00'] > 1:
continue # We only want one record at most
HEAD_ID = HEAD.get('Message_Identification')
HEAD_DATE = HEAD.get('Message_date')
HEAD_TIME = HEAD.get('Message_time')
HEAD_DATETIME = get_datetime(HEAD_DATE, HEAD_TIME)
HEAD_FROM = HEAD.get('Sender_Identification')
HEAD_TO = None
out.ta_info['frompartner'] = HEAD_FROM
out.ta_info['topartner'] = HEAD_TO
lout.put({'BOTSID':'inventory'},{'BOTSID':'header', 'msg_id': HEAD_ID})
lout.put({'BOTSID':'inventory'},{'BOTSID':'header', 'datetime': HEAD_DATETIME})
lout.put({'BOTSID':'inventory'},{'BOTSID':'header', 'partner_name': HEAD_FROM})
# == WAREHOUSE REFERENCE - SR70R ==
for WH in BODY.get('SR70R', []):
rec_counts['70'] = rec_counts.setdefault('70', 0) + 1
if rec_counts['70'] > 1:
continue # We only want one record at most
for WH_ADD in WH.get('SR72R', []):
rec_counts['72'] = rec_counts.setdefault('72', 0) + 1
WH_ADD = WH.get('SR72R', [{}])[0]
WH_PARTNER_ID = WH.get('Party_external_reference')
WH_ADD_NAME = WH_ADD.get('Company_name_1')
WH_ADD_NAME2 = WH_ADD.get('Company_name_2')
if WH_ADD_NAME2:
WH_ADD_NAME = WH_ADD_NAME + ", " + WH_ADD_NAME2
WH_ADD_LINE1, WH_ADD_LINE2, WH_ADD_LINE3 = get_address(WH_ADD.get('House_number'), WH_ADD.get('House_number_extension'), WH_ADD.get('Address_1'), WH_ADD.get('Address_2'), WH_ADD.get('Address_3'), WH_ADD.get('Address_4'))
WH_ADD_ZIP = WH_ADD.get('Postal_code')
WH_ADD_STATE = WH_ADD.get('Place_name')
WH_ADD_COUNTRY = WH_ADD.get('Country_code')
WH_ADD_PHONE = WH_ADD.get('Telephone_number')
WH_ADD_FAX = WH_ADD.get('Telefax_number')
WH_ADD_VAT = WH_ADD.get('VAT-number')
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'id': WH_PARTNER_ID})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'name': WH_ADD_NAME})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'street1': WH_ADD_LINE1})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'street2': WH_ADD_LINE2})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'city': WH_ADD_LINE3})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'state': WH_ADD_STATE})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'country': WH_ADD_COUNTRY})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'zip': WH_ADD_ZIP})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'phone': WH_ADD_PHONE})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'fax': WH_ADD_FAX})
lout.put({'BOTSID':'inventory'},{'BOTSID':'partner', 'vat': WH_ADD_VAT})
# == STOCK LEVELS - SR8AR ==
for SL in BODY.get('SR8AR', []):
rec_counts['8A'] = rec_counts.setdefault('8A', 0) + 1
SL_PRODUCT = SL.get('Lot_external_reference')
SL_PRODUCT_ARTICLE = SL.get('Article_reference')
SL_PRODUCT_CODE = SL.get('Product_code')
SL_PRODUCT_CODE_TYPE = SL.get('Product_code_qualifier')
SL_QTY_TOTAL = SL.get('Number_of_units_physical_stock_in_MUS')
SL_QTY_INCOMING = SL.get('Expected_in')
SL_QTY_AVAILABLE = SL.get('Physical_stock_available')
SL_QTY_OUTGOING = SL.get('Expected_out')
SL_QTY_OUTGOING_AVAILABLE = SL.get('Reserved_sufficient')
SL_QTY_OUTGOING_FUTURE = SL.get('Reserved_insufficient')
SL_DATE = SL.get('Date_the_stock_was_determined')
SL_TIME = SL.get('Time_the_stock_was_determined')
SL_DATETIME = get_datetime(SL_DATE, SL_TIME)
sout = lout.putloop({'BOTSID': 'inventory'}, {'BOTSID':'inventory_line'})
sout.put({'BOTSID':'inventory_line', 'product': SL_PRODUCT})
sout.put({'BOTSID':'inventory_line', 'product_article_no': SL_PRODUCT_ARTICLE})
sout.put({'BOTSID':'inventory_line', 'product_other': SL_PRODUCT_CODE})
sout.put({'BOTSID':'inventory_line', 'product_other_type': SL_PRODUCT_CODE_TYPE})
#sout.put({'BOTSID':'inventory_line', 'qty_total': SL_QTY_TOTAL}) # This may be in a different UoM from DSV, so do not include it since the other qty fields are mandatory
sout.put({'BOTSID':'inventory_line', 'qty_incoming': SL_QTY_INCOMING})
sout.put({'BOTSID':'inventory_line', 'qty_available': SL_QTY_AVAILABLE})
sout.put({'BOTSID':'inventory_line', 'qty_outgoing': SL_QTY_OUTGOING})
sout.put({'BOTSID':'inventory_line', 'qty_outgoing_available': SL_QTY_OUTGOING_AVAILABLE})
sout.put({'BOTSID':'inventory_line', 'qty_outgoing_future': SL_QTY_OUTGOING_FUTURE})
sout.put({'BOTSID':'inventory_line', 'datetime': SL_DATETIME})
# == FOOTER - SR99R ==
for FOOT in BODY.get('SR99R', []):
rec_counts['99'] = rec_counts.setdefault('99', 0) + 1
for FOOT in BODY.get('SR99R', []):
FOOT_REC_TYPE = FOOT.get('RecordType')
FOOT_REC_COUNT = FOOT.get('Total_Nbrs_of_Rec')
assert int(rec_counts.get(FOOT_REC_TYPE, 0)) == int(FOOT_REC_COUNT), "Expected %s of record %s, got %s" % (FOOT_REC_COUNT, FOOT_REC_TYPE, rec_counts.get(FOOT_REC_TYPE, 0))
return
def main(inn, out):
data = parse_dsv_xml(inn.root, SCHEMA)
data = simplify_dsv_xml(data)
for BODY in data['SR']:
lout = out.putloop({'BOTSID': 'inventory'})
rec_counts = {}
# == HEADER - SR00R ==
for HEAD in BODY.get('SR00R', []):
rec_counts['00'] = rec_counts.setdefault('00', 0) + 1
if rec_counts['00'] > 1:
continue # We only want one record at most
HEAD_ID = HEAD.get('Message_Identification')
HEAD_DATE = HEAD.get('Message_date')
HEAD_TIME = HEAD.get('Message_time')
HEAD_DATETIME = get_datetime(HEAD_DATE, HEAD_TIME)
HEAD_FROM = HEAD.get('Sender_Identification')
HEAD_TO = None
out.ta_info['frompartner'] = HEAD_FROM
out.ta_info['topartner'] = HEAD_TO
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'header',
'msg_id': HEAD_ID
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'header',
'datetime': HEAD_DATETIME
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'header',
'partner_name': HEAD_FROM
})
# == WAREHOUSE REFERENCE - SR70R ==
for WH in BODY.get('SR70R', []):
rec_counts['70'] = rec_counts.setdefault('70', 0) + 1
if rec_counts['70'] > 1:
continue # We only want one record at most
for WH_ADD in WH.get('SR72R', []):
rec_counts['72'] = rec_counts.setdefault('72', 0) + 1
WH_ADD = WH.get('SR72R', [{}])[0]
WH_PARTNER_ID = WH.get('Party_external_reference')
WH_ADD_NAME = WH_ADD.get('Company_name_1')
WH_ADD_NAME2 = WH_ADD.get('Company_name_2')
if WH_ADD_NAME2:
WH_ADD_NAME = WH_ADD_NAME + ", " + WH_ADD_NAME2
WH_ADD_LINE1, WH_ADD_LINE2, WH_ADD_LINE3 = get_address(
WH_ADD.get('House_number'),
WH_ADD.get('House_number_extension'), WH_ADD.get('Address_1'),
WH_ADD.get('Address_2'), WH_ADD.get('Address_3'),
WH_ADD.get('Address_4'))
WH_ADD_ZIP = WH_ADD.get('Postal_code')
WH_ADD_STATE = WH_ADD.get('Place_name')
WH_ADD_COUNTRY = WH_ADD.get('Country_code')
WH_ADD_PHONE = WH_ADD.get('Telephone_number')
WH_ADD_FAX = WH_ADD.get('Telefax_number')
WH_ADD_VAT = WH_ADD.get('VAT-number')
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'id': WH_PARTNER_ID
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'name': WH_ADD_NAME
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'street1': WH_ADD_LINE1
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'street2': WH_ADD_LINE2
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'city': WH_ADD_LINE3
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'state': WH_ADD_STATE
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'country': WH_ADD_COUNTRY
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'zip': WH_ADD_ZIP
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'phone': WH_ADD_PHONE
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'fax': WH_ADD_FAX
})
lout.put({'BOTSID': 'inventory'}, {
'BOTSID': 'partner',
'vat': WH_ADD_VAT
})
# == STOCK LEVELS - SR8AR ==
for SL in BODY.get('SR8AR', []):
rec_counts['8A'] = rec_counts.setdefault('8A', 0) + 1
SL_PRODUCT = SL.get('Lot_external_reference')
SL_PRODUCT_ARTICLE = SL.get('Article_reference')
SL_PRODUCT_CODE = SL.get('Product_code')
SL_PRODUCT_CODE_TYPE = SL.get('Product_code_qualifier')
SL_QTY_TOTAL = SL.get('Number_of_units_physical_stock_in_MUS')
SL_QTY_INCOMING = SL.get('Expected_in')
SL_QTY_AVAILABLE = SL.get('Physical_stock_available')
SL_QTY_OUTGOING = SL.get('Expected_out')
SL_QTY_OUTGOING_AVAILABLE = SL.get('Reserved_sufficient')
SL_QTY_OUTGOING_FUTURE = SL.get('Reserved_insufficient')
SL_DATE = SL.get('Date_the_stock_was_determined')
SL_TIME = SL.get('Time_the_stock_was_determined')
SL_DATETIME = get_datetime(SL_DATE, SL_TIME)
sout = lout.putloop({'BOTSID': 'inventory'},
{'BOTSID': 'inventory_line'})
sout.put({'BOTSID': 'inventory_line', 'product': SL_PRODUCT})
sout.put({
'BOTSID': 'inventory_line',
'product_article_no': SL_PRODUCT_ARTICLE
})
sout.put({
'BOTSID': 'inventory_line',
'product_other': SL_PRODUCT_CODE
})
sout.put({
'BOTSID': 'inventory_line',
'product_other_type': SL_PRODUCT_CODE_TYPE
})
#sout.put({'BOTSID':'inventory_line', 'qty_total': SL_QTY_TOTAL}) # This may be in a different UoM from DSV, so do not include it since the other qty fields are mandatory
sout.put({
'BOTSID': 'inventory_line',
'qty_incoming': SL_QTY_INCOMING
})
sout.put({
'BOTSID': 'inventory_line',
'qty_available': SL_QTY_AVAILABLE
})
sout.put({
'BOTSID': 'inventory_line',
'qty_outgoing': SL_QTY_OUTGOING
})
sout.put({
'BOTSID': 'inventory_line',
'qty_outgoing_available': SL_QTY_OUTGOING_AVAILABLE
})
sout.put({
'BOTSID': 'inventory_line',
'qty_outgoing_future': SL_QTY_OUTGOING_FUTURE
})
sout.put({'BOTSID': 'inventory_line', 'datetime': SL_DATETIME})
# == FOOTER - SR99R ==
for FOOT in BODY.get('SR99R', []):
rec_counts['99'] = rec_counts.setdefault('99', 0) + 1
for FOOT in BODY.get('SR99R', []):
FOOT_REC_TYPE = FOOT.get('RecordType')
FOOT_REC_COUNT = FOOT.get('Total_Nbrs_of_Rec')
assert int(rec_counts.get(FOOT_REC_TYPE, 0)) == int(
FOOT_REC_COUNT), "Expected %s of record %s, got %s" % (
FOOT_REC_COUNT, FOOT_REC_TYPE,
rec_counts.get(FOOT_REC_TYPE, 0))
return
def main(inn, out):
data = parse_dsv_xml(inn.root, SCHEMA)
data = simplify_dsv_xml(data)
for BODY in data["WOC"]:
lout = out.putloop({"BOTSID": "orderconf"})
rec_counts = {}
# == HEADER - WOC00R ==
for HEAD in BODY.get("WOC00R", []):
rec_counts["00"] = rec_counts.setdefault("00", 0) + 1
if rec_counts["00"] > 1:
continue # We only want one record at most
HEAD_ID = HEAD.get("Principal-ID")
HEAD_ID2 = HEAD.get("File-ID")
HEAD_DATE = HEAD.get("Message_date")
HEAD_TIME = HEAD.get("Message_time")
HEAD_DATETIME = get_datetime(HEAD_DATE, HEAD_TIME)
HEAD_FROM = HEAD.get("Principal_off_code")
HEAD_TO = None
HEAD_TEST = HEAD.get("Test_indicator")
HEAD_STATE = HEAD.get("Message_type")
HEAD_TYPE = HEAD.get("Message_type-ID")
out.ta_info["frompartner"] = HEAD_FROM
out.ta_info["topartner"] = HEAD_TO
out.ta_info["testindicator"] = HEAD_TEST == "1"
lout.put({"BOTSID": "orderconf"}, {"BOTSID": "header", "msg_id": HEAD_ID})
lout.put({"BOTSID": "orderconf"}, {"BOTSID": "header", "msg_id2": HEAD_ID2})
lout.put({"BOTSID": "orderconf"}, {"BOTSID": "header", "datetime": HEAD_DATETIME})
lout.put({"BOTSID": "orderconf"}, {"BOTSID": "header", "state": HEAD_STATE})
lout.put({"BOTSID": "orderconf"}, {"BOTSID": "header", "type": HEAD_TYPE})
lout.put({"BOTSID": "orderconf"}, {"BOTSID": "header", "test": out.ta_info["testindicator"]})
# == ORDER - WOC50R ==
for ORD in BODY.get("WOC50R", []):
rec_counts["50"] = rec_counts.setdefault("50", 0) + 1
ORD_ID = ORD.get("Order_reference")
ORD_NAME = ORD.get("Message_reference")
ORD_DESC = ORD.get("Order_remarks")
ORD_MANIFEST = ORD.get("Voyage_id")
ORD_SHIP_DATE = get_datetime(ORD.get("Shipped_on_this_date"), "0000")
ORD_DELIVERY_DATE = ORD.get("Delivery_date")
ORD_DELIVERY_DATE_TYPE = {"O": "ondate", "D": "fromdate"}.get(ORD.get("Code_delivery_date", "F"))
ORD_TYPE = {"I": "in", "O": "out", "C": "customs", "T": "skuchange", "X": "direct"}.get(
ORD.get("Order_type")
)
ORD_REASON = {
"B": "backorder",
"C": "correction",
"G": "refuse",
"H": "return",
"N": "normal",
"S": "companytransfer",
}.get(ORD.get("Order_category", "N"))
ORD_SERVICE_OUT = ORD.get("Ext_forw_service")
ORD_CARRIER_CODE = ORD.get("Carrier_code")
ORD_CONFIRMED = ORD.get("Indicator_Order_Executed")
ORD_INVOICE_NAME = ORD.get("Invoice_number")
ORD_INVOICE_CURRENCY = ORD.get("Currency_of_order")
ORD_INVOICE_PAYTERM = ORD.get("Payment_term")
ORD_B2_TYPE = {"B": "B2B", "C": "B2C"}.get(ORD.get("Sales_Channel"))
oout = lout.putloop({"BOTSID": "orderconf"}, {"BOTSID": "shipment"})
oout.put({"BOTSID": "shipment", "id": ORD_ID})
oout.put({"BOTSID": "shipment", "name": ORD_NAME})
oout.put({"BOTSID": "shipment", "desc": ORD_DESC})
oout.put({"BOTSID": "shipment", "manifest": ORD_MANIFEST})
oout.put({"BOTSID": "shipment", "date_ship": ORD_SHIP_DATE})
oout.put({"BOTSID": "shipment", "date_delivery": ORD_DELIVERY_DATE})
oout.put({"BOTSID": "shipment", "date_delivery_type": ORD_DELIVERY_DATE_TYPE})
oout.put({"BOTSID": "shipment", "type": ORD_TYPE})
oout.put({"BOTSID": "shipment", "reason": ORD_REASON})
oout.put({"BOTSID": "shipment", "service": ORD_SERVICE_OUT})
oout.put({"BOTSID": "shipment", "carrier": ORD_CARRIER_CODE})
oout.put({"BOTSID": "shipment", "confirmed": ORD_CONFIRMED})
oout.put({"BOTSID": "shipment", "buisness_type": ORD_B2_TYPE})
oout.put({"BOTSID": "shipment"}, {"BOTSID": "invoice", "name": ORD_INVOICE_NAME})
oout.put({"BOTSID": "shipment"}, {"BOTSID": "invoice", "currency": ORD_INVOICE_CURRENCY})
oout.put({"BOTSID": "shipment"}, {"BOTSID": "invoice", "payment_term": ORD_INVOICE_PAYTERM})
# == ORDER REFERENCES - WOC51R ==
for ORDID in ORD.get("WOC51R", []):
rec_counts["51"] = rec_counts.setdefault("51", 0) + 1
ORDID_ID = ORDID.get("Reference_number")
ORDID_DESC = ORDID.get("Free_form_descr")
ORDID_TYPE = {
"EPF": "invoice",
"CON": "consignment",
"ABO": "orig_ref",
"PO": "purchase_ref",
"SI": "shipping_ref",
"SO": "sale_ref",
}.get(ORDID.get("Reference_qualifier"))
ORDID_DATE = ORDID.get("Date")
ORDID_TIME = ORDID.get("Time")
ORDID_DATEIMTE = get_datetime(ORDID_DATE, ORDID_TIME)
oidout = oout.putloop({"BOTSID": "shipment"}, {"BOTSID": "references"})
oidout.put({"BOTSID": "references", "id": ORDID_ID})
oidout.put({"BOTSID": "references", "desc": ORDID_DESC})
oidout.put({"BOTSID": "references", "type": ORDID_TYPE})
oidout.put({"BOTSID": "references", "datetime": ORDID_DATEIMTE})
# == ORDER VALUES - WOC52R ==
for ORDVAL in ORD.get("WOC52R", []):
rec_counts["52"] = rec_counts.setdefault("52", 0) + 1
ORDVAL_CURRENCY = ORDVAL.get("Currency_code")
ORDID_TOTAL = ORDVAL.get("Order_total")
ORDVAL_TYPE = {
"CTN": "uoms",
"PLT": "pallets",
"GRW": "gross_grammes",
"NTW": "net_grammes",
"MTQ": "gross_cbm",
"IN": "invoice",
"F01": "proforma_exvat",
}.get(ORDVAL.get("Qualifier"))
ovalout = oout.putloop({"BOTSID": "shipment"}, {"BOTSID": "values"})
ovalout.put({"BOTSID": "values", "type": ORDVAL_TYPE})
ovalout.put({"BOTSID": "values", "total": ORDID_TOTAL})
ovalout.put({"BOTSID": "values", "currency": ORDVAL_CURRENCY})
# == PAL DETAILS - WOC54R ==
for ORDLD in ORD.get("WOC54R", []):
rec_counts["54"] = rec_counts.setdefault("54", 0) + 1
# FIXME: Unused but count for validation
# == BOX DETAILS - WOC56R ==
for ORDLD in ORD.get("WOC56R", []):
rec_counts["56"] = rec_counts.setdefault("56", 0) + 1
# FIXME: Unused but count for validation
# == SERVICE DETAILS - WOC59R ==
for ORDLD in ORD.get("WOC59R", []):
rec_counts["59"] = rec_counts.setdefault("59", 0) + 1
CARRIER_TYPE = {
"A": "Allport",
"B": "DPD",
"C": "Coca Cola",
"D": "Dachser",
"E": "Exel",
"F": "Dufa transport",
"G": "Graveleau",
"H": "DHL",
"I": "Chronopost",
"J": "XP France",
"K": "Selektvracht",
"L": "Distrilogistics",
"M": "DSV (Frans Maas)",
"N": "Normal",
"O": "DSV",
"P": "TPG",
"Q": "Mory",
"R": "Nunner",
"S": "Sernam",
"T": "TNT",
"U": "UPS",
"V": "van Gend & Loos",
"W": "MIT Transport",
"X": "Fedex",
"Y": "New Poney Express",
"Z": "CORPCO",
"1": "DSV Road",
"2": "CIBLEX",
"3": "Trans-o-Flex",
"4": "AZKAR",
"5": "NightFreight",
"6": "GLS",
"7": "Bosman",
"8": "XPACK",
"9": "Ship It Smarter",
"(": "GEPC Intercompany",
")": "DynaLogic",
"-": "DB Schenker",
}.get(ORDLD.get("Carrier"))
SERVICE_LEVEL_CODE = ORDLD.get("Service_Level_Code")
CONSIGNMENT_NUMBER = ORDLD.get("Consignment_number")
SERVICE_LEVEL_DESC = ORDLD.get("Service_Level_Description")
SERVICE_LEVEL_LABEL = ORDLD.get("Service_Level_Description_Label")
SERVICE_LEVEL_TEXT = ORDLD.get("Service_Level_Description_Text")
SERVICE_IND_1 = ORDLD.get("Service_Indicator_01") or ""
SERVICE_IND_2 = ORDLD.get("Service_Indicator_02") or ""
SERVICE_IND_3 = ORDLD.get("Service_Indicator_03") or ""
LEAD_TRACKING_NUMBER = ORDLD.get("Lead_package_tracking_number")
oout.put({"BOTSID": "shipment", "service_carrier": CARRIER_TYPE})
oout.put({"BOTSID": "shipment", "service_level_code": SERVICE_LEVEL_CODE})
oout.put({"BOTSID": "shipment", "service_level_desc": SERVICE_LEVEL_DESC})
oout.put({"BOTSID": "shipment", "service_level_label": SERVICE_LEVEL_LABEL})
oout.put({"BOTSID": "shipment", "service_level_text": SERVICE_LEVEL_TEXT})
oout.put(
{
"BOTSID": "shipment",
"service_indicators": "%s,%s,%s" % (SERVICE_IND_1, SERVICE_IND_2, SERVICE_IND_3),
}
)
oout.put({"BOTSID": "shipment", "consignment_number": LEAD_TRACKING_NUMBER})
oout.put({"BOTSID": "shipment", "tracking_number": LEAD_TRACKING_NUMBER})
# == PARTNER REFERENCE - WOC70R ==
for PART in ORD.get("WOC70R", []):
rec_counts["70"] = rec_counts.setdefault("70", 0) + 1
for PART_ADD in PART.get("WOC72R", []):
rec_counts["72"] = rec_counts.setdefault("72", 0) + 1
PART_ADD = PART.get("WOC72R", [{}])[0]
PART_PARTNER_ID = PART.get("Party_external_reference")
PART_ADD_NAME = PART_ADD.get("Company_name_1")
PART_ADD_NAME2 = PART_ADD.get("Company_name_2")
if PART_ADD_NAME2:
PART_ADD_NAME = PART_ADD_NAME + ", " + PART_ADD_NAME2
PART_ADD_LINE1, PART_ADD_LINE2, PART_ADD_LINE3 = get_address(
PART_ADD.get("House_number"),
PART_ADD.get("House_number_extension"),
PART_ADD.get("Address_1"),
PART_ADD.get("Address_2"),
PART_ADD.get("Address_3"),
PART_ADD.get("Address_4"),
)
PART_ADD_ZIP = PART_ADD.get("Postal_code")
PART_ADD_STATE = PART_ADD.get("Place_name")
PART_ADD_COUNTRY = PART_ADD.get("Country_code")
PART_ADD_PHONE = PART_ADD.get("Telephone_number")
PART_ADD_FAX = PART_ADD.get("Telefax_number")
PART_ADD_VAT = PART_ADD.get("VAT-number")
partout = oout.putloop({"BOTSID": "shipment"}, {"BOTSID": "partner"})
partout.put({"BOTSID": "partner", "id": PART_PARTNER_ID})
partout.put({"BOTSID": "partner", "name": PART_ADD_NAME})
partout.put({"BOTSID": "partner", "street1": PART_ADD_LINE1})
partout.put({"BOTSID": "partner", "street2": PART_ADD_LINE2})
partout.put({"BOTSID": "partner", "city": PART_ADD_LINE3})
partout.put({"BOTSID": "partner", "state": PART_ADD_STATE})
partout.put({"BOTSID": "partner", "country": PART_ADD_COUNTRY})
partout.put({"BOTSID": "partner", "zip": PART_ADD_ZIP})
partout.put({"BOTSID": "partner", "phone": PART_ADD_PHONE})
partout.put({"BOTSID": "partner", "fax": PART_ADD_VAT})
partout.put({"BOTSID": "partner", "vat": PART_ADD_VAT})
# == ORDER LINES - WOC80R ==
for ORDL in ORD.get("WOC80R", []):
rec_counts["80"] = rec_counts.setdefault("80", 0) + 1
ORDL_DATE = HEAD.get("Date_executed")
ORDL_TIME = HEAD.get("Time_executed")
ORDL_PICK_DATE = HEAD.get("Picking_date")
ORDL_PICK_TIME = HEAD.get("Picking_time")
# take the one that is more precise with the picking time having precedence
if ORDL_PICK_DATE and ORDL_PICK_TIME:
ORDL_DATETIME = get_datetime(ORDL_PICK_DATE, ORDL_PICK_TIME)
elif ORDL_DATE and ORDL_TIME:
ORDL_DATETIME = get_datetime(ORDL_DATE, ORDL_TIME)
else:
ORDL_DATETIME = get_datetime(ORDL_PICK_DATE, ORDL_PICK_TIME) or get_datetime(ORDL_DATE, ORDL_TIME)
ORDL_ID = ORDL.get("Orderline_reference")
ORDL_SEQ = ORDL.get("FL_order_line_nr")
ORDL_TYPE = {"I": "in", "O": "out"}.get(ORDL.get("Orderline-type"))
ORDL_PRODUCT = ORDL.get("SKU-reference")
ORDL_QTY_UOM = ORDL.get("Quantity")
ORDL_UOM = ORDL.get("Unit_of_Measure")
ORDL_QTY_REAL = ORDL.get("Packages") or ORDL.get("Units_MUS")
ORDL_QTY_EXPECTED = ORDL.get("Original_packages") or ORDL.get("Original_MUS")
olout = oout.putloop({"BOTSID": "shipment"}, {"BOTSID": "line"})
olout.put({"BOTSID": "line", "id": ORDL_ID})
olout.put({"BOTSID": "line", "seq": ORDL_SEQ})
olout.put({"BOTSID": "line", "type": ORDL_TYPE})
olout.put({"BOTSID": "line", "datetime": ORDL_DATETIME})
olout.put({"BOTSID": "line", "product": ORDL_PRODUCT})
olout.put({"BOTSID": "line", "uom_qty": ORDL_QTY_UOM})
olout.put({"BOTSID": "line", "uom": ORDL_UOM})
olout.put({"BOTSID": "line", "qty_real": ORDL_QTY_REAL})
olout.put({"BOTSID": "line", "qty_expected": ORDL_QTY_EXPECTED})
# == ORDER LINE REFERENCES - WOC81R ==
for ORDLID in ORDL.get("WOC81R", []):
rec_counts["81"] = rec_counts.setdefault("81", 0) + 1
ORDLID_ID = ORDLID.get("Reference_number")
ORDLID_DESC = ORDLID.get("Free_form_descr")
ORDLID_TYPE = {
"EPF": "invoice",
"CON": "consignment",
"ABO": "orig_ref",
"PO": "purchase_ref",
"SI": "shipping_ref",
"SO": "sale_ref",
"BAT": "batch_seq",
"PBI": "pick_batch",
"SO": "pick_seq",
"SO": "kit",
}.get(ORDLID.get("Reference_qualifier"))
oidout = olout.putloop({"BOTSID": "line"}, {"BOTSID": "references"})
oidout.put({"BOTSID": "references", "id": ORDLID_ID})
oidout.put({"BOTSID": "references", "desc": ORDLID_DESC})
oidout.put({"BOTSID": "references", "type": ORDLID_TYPE})
# == ORDER LINE DETAILS - WOC83R ==
for ORDLD in ORDL.get("WOC83R", []):
rec_counts["83"] = rec_counts.setdefault("83", 0) + 1
# FIXME: Unused but count for validation
# == ORDER LINE DETAILS - WOC85R ==
for ORDLD in ORDL.get("WOC85R", []):
rec_counts["85"] = rec_counts.setdefault("85", 0) + 1
# FIXME: Unused but count for validation
for FOOT in BODY.get("WOC99R", []):
rec_counts["99"] = rec_counts.setdefault("99", 0) + 1
for FOOT in BODY.get("WOC99R", []):
FOOT_REC_TYPE = FOOT.get("RecordType")
FOOT_REC_COUNT = FOOT.get("Total_Nbrs_of_Rec")
assert int(rec_counts.get(FOOT_REC_TYPE, 0)) == int(FOOT_REC_COUNT), "Expected %s of record %s, got %s" % (
FOOT_REC_COUNT,
FOOT_REC_TYPE,
rec_counts.get(FOOT_REC_TYPE, 0),
)
return
def main_input(message): #工具扫描主类
global params_sub
start_date = util.get_datetime() #开始时间
config.load_properties()
config.get_stream_name_and_tool(message)
tool_type = config.properties_info['TOOL_TYPE']
stream_name = config.properties_info['STREAM_NAME']
#load multi tool properties
config.load_mutil_tool_properties(tool_type)
#map CodeCC API data
config.map_properties_info(offline_properties_info)
#map command properties info
config.map_offline_properties_info(offline_properties_info)
#verify all properties
# config.verify_info()
#update the properties
config.properties_update()
config.generate_config(config.properties_info)
if "SUB_PATH" in config.properties_info:
os.environ["PATH"] = config.properties_info[
'SUB_PATH'] + os.pathsep + os.environ["PATH"]
#开始清空pid文件
pid_config.clean_pid(config.properties_info["PID_FILE"])
#添加自身pid
pid_config.add_pid(str(os.getpid()), config.properties_info["PID_FILE"])
if 'OFFLINE' in config.properties_info:
#排队开始
params_sub = {
'stepNum': str(1),
'startTime': str(int(time.time())),
'endTime': '0',
'msg': '',
'flag': 3
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
#排队结束
params_sub = {
'stepNum': str(1),
'startTime': '0',
'endTime': str(int(time.time())),
'msg': '',
'flag': 1
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
#下载开始
params_sub = {
'stepNum': str(2),
'startTime': str(int(time.time())),
'endTime': '0',
'msg': '',
'flag': 3
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
#download source code
latest_info = main.download_code(config.properties_info)
#下载结束
params_sub = {
'stepNum': str(2),
'startTime': '0',
'endTime': str(int(time.time())),
'msg': str(latest_info),
'flag': 1
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
#git项目，获取本地remote url和branch
config.properties_git_info_update()
#扫描开始
scan_finish_message = ""
params_sub = {
'stepNum': str(3),
'startTime': str(int(time.time())),
'endTime': '0',
'msg': '',
'flag': 3
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
if tool_type == "cpplint":
if "SUB_PATH" in config.properties_info and 'PY27_PATH' in config.properties_info:
config.properties_info['SUB_PATH'] = config.properties_info[
"PY27_PATH"] + os.pathsep + config.properties_info["SUB_PATH"]
file.skip(config.properties_info)
elif tool_type == "pylint" or tool_type == "eslint" or tool_type == "checkstyle" or \
tool_type == "stylecop" or tool_type == "ccn" or tool_type == "detekt" or \
tool_type == "sensitive" or tool_type == "phpcs" or tool_type == "occheck" or \
tool_type == "spotbugs":
if "SUB_PATH" in config.properties_info and 'PY35_PATH' in config.properties_info:
config.properties_info['SUB_PATH'] = config.properties_info[
"PY35_PATH"] + os.pathsep + config.properties_info["SUB_PATH"]
file.skip(config.properties_info)
elif tool_type == "dupc" or tool_type == "goml":
if "SUB_PATH" in config.properties_info and 'PY35_PATH' in config.properties_info:
config.properties_info['SUB_PATH'] = config.properties_info[
"PY35_PATH"] + os.pathsep + config.properties_info["SUB_PATH"]
main.main_scan(config.properties_info)
#print(util.get_datetime()+" "+tool_type+" generate scm blame and info")
scm.generate_blame_and_info(config.properties_info)
if tool_type == "goml" and os.path.exists(
config.properties_info['STREAM_DATA_PATH'] + '/go_build.log'):
with open(config.properties_info['STREAM_DATA_PATH'] + '/go_build.log',
"r",
encoding='utf-8') as go_build_file:
scan_finish_message = go_build_file.read()
#扫描结束
params_sub = {
'stepNum': str(3),
'startTime': '0',
'endTime': str(int(time.time())),
'msg': scan_finish_message,
'flag': 1
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
#提交开始
params_sub = {
'stepNum': str(4),
'startTime': str(int(time.time())),
'endTime': '0',
'msg': '',
'flag': 3
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
#json the scan data
if os.path.exists(config.properties_info["STREAM_DATA_PATH"]):
#print(util.get_datetime()+" "+tool_type+" generate json from stream data")
#提交数据
if tool_type == "dupc":
main.dupc_generate_data_json(config.properties_info)
else:
main.generate_data_json(config.properties_info)
#提交平均圈复杂度数
if tool_type == "ccn":
codecc_web.codecc_upload_avg_ccn(
stream_name, config.properties_info['TASK_ID'],
config.properties_info['PROJECT_AVG_FILE_CC_LIST'])
#提交结束
params_sub = {
'stepNum': str(4),
'startTime': '0',
'endTime': str(int(time.time())),
'msg': '',
'flag': 1
}
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
print(config)
if not 'IGNORE_DELETE_LOG' in config.properties_info:
#删除临时文件
private_key = "/tmp/." + stream_name + '_' + tool_type + "_private_key"
file.delete_file_folder(private_key)
if os.path.exists(config.properties_info["STREAM_DATA_PATH"]):
file.delete_file_folder(config.properties_info["STREAM_DATA_PATH"])
#结束清空pid文件
pid_config.clean_pid(config.properties_info["PID_FILE"])
finish_date = util.get_datetime()
print(tool_type + ' scan finish: ' + start_date + ' to ' + finish_date)
def get_player_info(url):
"""Fetch a player's information from bbref and return a dict of dicts containing personal and player information"""
player_response = requests.get("https://www.basketball-reference.com" +
url)
player_soup = bs4.BeautifulSoup(player_response.text, "html.parser")
person = {}
player = {}
image_url = player_soup.find(attrs={"itemscope": "image"})
if image_url is not None:
player["image_url"] = image_url["src"]
player_info = player_soup.find(
attrs={"itemtype": "https://schema.org/Person"})
name = player_info.find("h1").get_text().split(" ")
player_info = player_info("p")
person["preferred_name"] = name[0]
if len(name) < 2:
name.append("Hilario")
last_name = name[1]
raw_info = []
for info in player_info:
raw_info.append(info.get_text().replace("\n", " "))
ignore = [
"Pronunciation", "High School", "Hall of Fame", "Draft", "Experience",
"Relatives", "Died", "Recruiting", "(born"
]
for info in raw_info:
if any(string in info for string in ignore):
continue
# clean up info
info = info.replace("\xa0", " ").split(" ")
info_arr = [x for x in info if x]
if last_name in info_arr:
full_name = " ".join(info_arr).split("▪")[0].split(" ")
last_name_idx = full_name.index(last_name)
person["last_name"] = " ".join(
full_name[last_name_idx:]).strip().strip("I").strip().replace(
"Jr.", "").strip()
person["middle_name"] = " ".join(
full_name[1:last_name_idx]).strip()
person["first_name"] = full_name[0]
# players with 3 names (and James Michael McAdoo)
special_names = [
"Jo English", "Rod Hundley", "Carlos Navarro", "John Ramos",
"Ray Richardson", "Michael Ray McAdoo", "Vander Velden",
"Rod Williams"
]
if person["last_name"] in special_names or re.search(
r"^[vV][ao]n ", person["last_name"]):
names = person["last_name"].split(" ")
person["middle_name"] = " ".join(names[0:-1])
person["preferred_name"] = person["preferred_name"] + \
" " + names[0]
person["last_name"] = names[-1]
if "Position:" in info_arr:
positions = {
"Point": "PG",
"Shooting": "SG",
"Small": "SF",
"Power": "PF",
"Center": "C",
"Guard": "G",
"Forward": "F"
}
player["shooting_hand"] = info_arr[-1]
block_idx = info_arr.index("▪")
position_string = " ".join(info_arr[1:block_idx])
position_list = re.findall(r"[\w']+",
" ".join(position_string.split("and")))
player_positions = set()
for pos in position_list:
player_positions.add(positions[pos])
player["positions"] = list(player_positions)
if "kg" in " ".join(info_arr):
player["weight"] = float(info_arr[-1].split("kg")[0])
player["height"] = float(info_arr[-2].split("cm")[0][1:])
if "Born:" in info_arr:
birth_date = " ".join(info_arr[1:4])
person["dob"] = birth_date
if len(info_arr) > 4:
in_idx = info_arr.index("in")
person["birth_place"] = " ".join(info_arr[in_idx + 1:-1])
if "College:" in info_arr or "Colleges:" in info_arr:
person["college"] = " ".join(info_arr[1:])
if "Debut:" in info_arr:
debut_info = " ".join(info_arr).split("&blacksquare")
if len(debut_info) > 1:
debut_years = [info.split(" ")[-1] for info in debut_info]
if debut_years[0] < debut_years[1]:
debut_idx = 0
else:
debut_idx = 1
debut_info = debut_info[debut_idx].strip().split(" ")
debut_date = get_datetime(debut_info[-1].split(":")[-1])
player["aba"] = True
else:
debut_date = get_datetime(debut_info[-1].split(":")[-1])
player["aba"] = False
rookie_year = debut_date.year + 1
if debut_date.month < 7:
rookie_year -= 1
player["rookie_season"] = rookie_year
player_seasons = player_soup.find_all(attrs={
"data-stat": "season",
"scope": "row",
"class": "left"
})
player_seasons = [season for season in player_seasons if season("a")]
final_season = player_seasons[-1].get_text()
final_season = int(final_season[0:2] + final_season[-2:])
if final_season == 1900:
final_season = 2000
if final_season != 2019:
player["final_season"] = final_season
return {"player": player, "person": person}
def test_next_datetime(): ret = '' arr = ['0300', '0900', '1200', '1800'] ret += '\n' ret += "['0300', '0900', '1200', '1800']\n" ret += '\n' ret += 'now\n' ret += str(util.next_datetime(arr)) + '\n' ret += '\n' dtstr = '2019-02-28 00:00:00.0000' ret += dtstr + '\n' ret += '(exp=03:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 03:00:00.0000' ret += dtstr + '\n' ret += '(exp=02-28 03:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 03:00:00.0001' ret += dtstr + '\n' ret += '(exp=02-28 09:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 09:00:00.0000' ret += dtstr + '\n' ret += '(exp=02-28 09:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 10:00:00.0000' ret += dtstr + '\n' ret += '(exp=02-28 12:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 15:00:00.0000' ret += dtstr + '\n' ret += '(exp=02-28 18:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 19:00:00.0000' ret += dtstr + '\n' ret += '(exp=03-01 03:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, moment=dt)) + '\n' # 2019-02-28 00:00:00.0000 ret += '\n' dt = 1551279600.000 ret += str(dt) + '\n' ret += '(exp=02-28 03:00)\n' ret += str(util.next_datetime(arr, moment=dt)) + '\n' # 2019-02-28 00:00:00.0000 ret += '\n' dt = 1551279600 ret += str(dt) + '\n' ret += '(exp=02-28 03:00)\n' ret += str(util.next_datetime(arr, moment=dt)) + '\n' # 2019-02-28 10:00:00.0000 ret += '\n' dt = 1551315600.000 ret += str(dt) + '\n' ret += '(exp=02-28 12:00)\n' ret += str(util.next_datetime(arr, moment=dt)) + '\n' # 2019-02-28 10:00:00.0000 ret += '\n' dt = 1551315600 ret += str(dt) + '\n' ret += '(exp=02-28 12:00)\n' ret += str(util.next_datetime(arr, moment=dt)) + '\n' ret += '----' ret += '\n' dtstr = '2019-02-28 00:00:00.0000' ret += dtstr + '\n' ret += '(-1 exp=02-27 18:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, -1, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 04:00:00.0000' ret += dtstr + '\n' ret += '(-1 exp=02-28 03:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, -1, moment=dt)) + '\n' ret += '----' ret += '\n' dtstr = '2019-02-28 00:00:00.0000' ret += dtstr + '\n' ret += '(-2 exp=02-27 12:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, -2, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 04:00:00.0000' ret += dtstr + '\n' ret += '(-2 exp=02-27 18:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, -2, moment=dt)) + '\n' ret += '----' ret += '\n' dtstr = '2019-02-28 00:00:00.0000' ret += dtstr + '\n' ret += '(2 exp=02-28 09:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, 2, moment=dt)) + '\n' ret += '\n' dtstr = '2019-02-28 04:00:00.0000' ret += dtstr + '\n' ret += '(2 exp=02-28 12:00)\n' dt = util.get_datetime(dtstr) ret += str(util.next_datetime(arr, 2, moment=dt)) + '\n' return ret
def add_timestamp(df, dt_field, ts_field):
df = df.dropna(subset=[dt_field])
df[ts_field] = map(lambda dt: time.mktime(get_datetime(dt).timetuple()),
df[dt_field])
return df
def main(inn, out):
root = {}
rec_counts = {}
# == HEADER - WSO00R ==
if True:
ELEMENT = "WSO00R"
SEGMENT = "00"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
HEAD_ID = inn.get({'BOTSID': 'picking'}, {
'BOTSID': 'header',
'message_id': None
})
HEAD_DATE, HEAD_TIME = get_datetime(
inn.get({'BOTSID': 'picking'}, {
'BOTSID': 'header',
'date_msg': None
}))
HEAD_TYPE = inn.get({'BOTSID': 'picking'}, {
'BOTSID': 'header',
'type': None
})
assert HEAD_TYPE == 'out', "Warehouse Shipping Orders must have type 'out', not '%s'" % (
HEAD_TYPE, )
HEAD_TYPE = "940" # Warehouse Shipping Order
d = OrderedDict([
('Segment_Number', SEGMENT),
('Identification_Nbr', HEAD_ID or ''),
('Message_Date', HEAD_DATE or ''),
('Message_Time', HEAD_TIME or ''),
('Message_Type_Id', HEAD_TYPE),
])
root.setdefault(ELEMENT, []).append(d)
pinn = inn.getloop({'BOTSID': 'picking'}, {'BOTSID': 'pickings'})
for pick in pinn:
ORD_ID = pick.get({'BOTSID': 'pickings', 'id': None})
assert ORD_ID, "Order ID must be present"
ORD_ID = re.sub(r'[\\/_-]', r'', ORD_ID.upper())
out.ta_info['botskey'] = ORD_ID # Set the botskey to the last order ID
ORD_STATE = pick.get({'BOTSID': 'pickings', 'state': None})
# == Cancelled Orders - WSO05R ==
if ORD_STATE == 'delete':
ELEMENT = "WSO05R"
SEGMENT = "05"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Order_reference', ORD_ID),
])
root.setdefault(ELEMENT, []).append(d)
# == New Orders - WSO10R ==
elif ORD_STATE == 'new':
ELEMENT = "WSO10R"
SEGMENT = "10"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
INCOTERM = pick.get({'BOTSID': 'pickings', 'incoterm': None})
ORD_PRIO = pick.get({'BOTSID': 'pickings', 'prio': None})
ORD_REMARK = pick.get({'BOTSID': 'pickings', 'desc': None})
ORD_DELIVERY_DATE, dummy = get_datetime(
pick.get({
'BOTSID': 'pickings',
'date': None
}) + ' 00:00:00.00000')
ord_root = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Type', 'O'), # Outgoing
('Indicator_Transport', 'Y'),
('Order_Reference', ORD_ID),
('Indicator_Labelling', 'Y'),
('Indicator_contact_cl', 'Y'),
('Terms_of_Delivery', INCOTERM),
('COD_amount', '0.0'),
('Currency_COD-amount', ''),
('Order_Category', 'N'), # N = New
('Order_priority', ORD_PRIO),
('Delivery_date', ORD_DELIVERY_DATE),
(
'Remark', ORD_REMARK
), # FIXME: We add this last to validate against DSV's example messages, it should come just before priority
])
root.setdefault(ELEMENT, []).append(ord_root)
# == Order Remarks - WSO20R ==
if ORD_REMARK:
ELEMENT = "WSO20R"
SEGMENT = "20"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Remark', ORD_REMARK),
('Order_Remark_Qual', 'D'), # Delivery Instructions
])
ord_root.setdefault(ELEMENT, []).append(d)
# == Order Party - WSO30R ==
ELEMENT = "WSO30R"
SEGMENT = "30"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
PART_ID = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'id': None
})
CLIENT_REF = pick.get({
'BOTSID': 'pickings',
'client_order_ref': None
})
PART_EMAIL = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'email': None
})
party_root = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Party_Qual',
'CN'), # Consignee - Party receiving goods
('Party_External_Ref', PART_ID),
('Client_Ref_to_Order', CLIENT_REF),
])
ord_root.setdefault(ELEMENT, []).append(party_root)
# == Order Party Email - WSO31R ==
if PART_EMAIL:
ELEMENT = "WSO31R"
SEGMENT = "31"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Communication_Number_Qualifier', 'EA'), # Email
('Communication_Number', PART_EMAIL),
('Recipient_type', 0), # 0=To, 1=CC, 2=BCC
])
party_root.setdefault(ELEMENT, []).append(d)
# == Order Party Address - WSO32R ==
ELEMENT = "WSO32R"
SEGMENT = "32"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
PART_NAME = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'name': None
}) or ''
PART_STREET1 = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'street1': None
}) or ''
PART_STREET2 = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'street2': None
}) or ''
PART_CITY = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'city': None
}) or ''
PART_ZIP = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'zip': None
}) or ''
PART_COUNTRY = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'country': None
}) or ''
PART_STATE = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'state': None
}) or ''
PART_PHONE = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'phone': None
}) or ''
PART_FAX = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'fax': None
}) or ''
PART_LANG = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'language': None
}) or ''
PART_VAT = pick.get({'BOTSID': 'pickings'}, {
'BOTSID': 'partner',
'vat': None
}) or ''
# Spill the first address line over to the second if too long, else truncate
if len(PART_STREET1) > 35 and not PART_STREET2:
space = PART_STREET1.rfind(' ', 0, 35)
if space:
OLD_STREET1 = PART_STREET1
PART_STREET1 = OLD_STREET1[:space]
PART_STREET2 = OLD_STREET1[space + 1:]
PART_STREET1 = PART_STREET1[:35]
PART_STREET2 = PART_STREET2[:35]
d = OrderedDict([
('Segment_Number', SEGMENT),
('Company_Name_1', PART_NAME[:35]),
('Address_3', PART_STREET2),
('Postal_Code', PART_ZIP),
('Place_Name', PART_CITY[:35]),
('Country_Code', PART_COUNTRY),
('Language_code', get_dsv_lang(PART_LANG)),
('Telephone_Number', PART_PHONE),
('Telefax_Number', PART_FAX),
('VAT-number', PART_VAT),
('Address_2', PART_STREET1),
('Address_4', PART_STATE),
])
party_root.setdefault(ELEMENT, []).append(d)
plines = pick.getloop({'BOTSID': 'pickings'}, {'BOTSID': 'line'})
for pline in plines:
# == Order Line - WSO40R ==
ELEMENT = "WSO40R"
SEGMENT = "40"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
LINE_ID = pline.get({'BOTSID': 'line', 'id': None})
LINE_SEQ = "%03d" % (int(
pline.get({
'BOTSID': 'line',
'seq': None
})), )
LINE_PRODUCT = pline.get({
'BOTSID': 'line',
'product': None
}).upper()
LINE_TYPE = '*FIRST'
LINE_QTY = pline.get({'BOTSID': 'line', 'product_qty': None})
LINE_DESC = pline.get({'BOTSID': 'line', 'desc': None})
LINE_VOLUME_NET = pline.get({
'BOTSID': 'line',
'volume_net': None
})
LINE_WEIGHT = pline.get({'BOTSID': 'line', 'weight': None})
LINE_WEIGHT_NET = pline.get({
'BOTSID': 'line',
'weight_net': None
})
LINE_PRICE_UNIT = pline.get({
'BOTSID': 'line',
'price_unit_ex_vat': None
}) or 0.0
LINE_CURRENCY = pline.get({
'BOTSID': 'line',
'price_currency': None
})
line_root = OrderedDict([
('Segment_Number', SEGMENT),
('Orderline_ID', LINE_SEQ),
('SKU_Reference', LINE_PRODUCT),
('Type_of_package', LINE_TYPE),
('Number_of_packages', LINE_QTY),
('Number_of_units_measure_unit_specifier', LINE_QTY),
('Number_of_units_measure_unit_specifier_per_type_of_package',
1), # FIXME: We only support units UoM currently
('Orderline_remarks', LINE_DESC),
('Net_volume_per_unit_measure_unit-specifier',
LINE_VOLUME_NET),
])
ord_root.setdefault(ELEMENT, []).append(line_root)
# == Order Line Invoice - WSO41R ==
ELEMENT = "WSO41R"
SEGMENT = "41"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
assert (
ord_root.get('Currency_COD-amount', LINE_CURRENCY)
or LINE_CURRENCY
) == LINE_CURRENCY, 'Multiple currencies per order is not supported'
ord_root.update({'Currency_COD-amount': LINE_CURRENCY})
try:
LINE_PRICE_UNIT = "%.2f" % (round(float(LINE_PRICE_UNIT),
2), )
except ValueError:
LINE_PRICE_UNIT = ""
d = OrderedDict([
('Segment_Number', SEGMENT),
('Goods_invoice_amount', LINE_PRICE_UNIT),
('Goods_inv_amoun_Qual', 'U'), # Unit price
])
line_root.setdefault(ELEMENT, []).append(d)
# Catch all
else:
raise NotImplementedError('Unable to handle order with state %s' %
(ORD_STATE, ))
# == FOOTER - WSO99R ==
rec_counts['99'] = len(rec_counts.keys())
for FOOT_REC_TYPE, FOOT_REC_COUNT in rec_counts.iteritems():
ELEMENT = "WSO99R"
SEGMENT = "99"
d = OrderedDict([
('Segment_Number', SEGMENT),
('RecordType', FOOT_REC_TYPE),
('Total_Nbrs_of_Rec', FOOT_REC_COUNT),
])
root.setdefault(ELEMENT, []).append(d)
root = OrderedDict(sorted(root.items(), key=lambda t: t[0]))
document = OrderedDict([('WSO', root)])
data = serialize_dsv_xml(document, SCHEMA)
out.root = data
return
def main(inn,out):
root = {}
rec_counts = {}
# == HEADER - WEO00R ==
if True:
ELEMENT = "WEO00R"
SEGMENT = "00"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
HEAD_ID = inn.get({'BOTSID': 'picking'}, {'BOTSID': 'header', 'message_id': None})
HEAD_DATE, HEAD_TIME = get_datetime(inn.get({'BOTSID': 'picking'}, {'BOTSID': 'header', 'date_msg': None}))
HEAD_TYPE = inn.get({'BOTSID': 'picking'}, {'BOTSID': 'header', 'type': None})
assert HEAD_TYPE == 'in', "Warehouse Entry Orders must have type 'in', not '%s'" % (HEAD_TYPE,)
HEAD_TYPE = "943" # Warehouse Entry Order
d = OrderedDict([
('Segment_Number', SEGMENT),
('Identification_Nbr', HEAD_ID or ''),
('Message_Date', HEAD_DATE or ''),
('Message_Time', HEAD_TIME or ''),
('Message_Type_Id', HEAD_TYPE),
])
root.setdefault(ELEMENT, []).append(d)
pinn = inn.getloop({'BOTSID': 'picking'}, {'BOTSID': 'pickings'})
for pick in pinn:
ORD_ID = pick.get({'BOTSID': 'pickings', 'id': None})
assert ORD_ID, "Order ID must be present"
ORD_ID = re.sub(r'[\\/_-]', r'', ORD_ID.upper())
ORD_STATE = pick.get({'BOTSID': 'pickings', 'state': None})
# == Cancelled Orders - WEO05R ==
if ORD_STATE == 'delete':
ELEMENT = "WEO05R"
SEGMENT = "05"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Reference', ORD_ID),
])
root.setdefault(ELEMENT, []).append(d)
# == New Orders - WEO10R ==
elif ORD_STATE == 'new':
ELEMENT = "WEO10R"
SEGMENT = "10"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
ORD_REMARK = pick.get({'BOTSID': 'pickings', 'desc': None})
ORD_DELIVERY_DATE, dummy = get_datetime(pick.get({'BOTSID': 'pickings', 'date': None}) + ' 00:00:00.00000')
ord_root = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Type', 'I'), # Incoming
('Indicator_Transport', 'N'),
('Order_Reference', ORD_ID),
('Indicator_Labelling', 'N'),
('Indicator_contact_cl', 'N'),
('Order_Category', 'N'), # N = New
('Date_expected', ORD_DELIVERY_DATE),
('Remark', ORD_REMARK),
])
root.setdefault(ELEMENT, []).append(ord_root)
# == Order Remarks - WEO20R ==
if ORD_REMARK:
ELEMENT = "WEO20R"
SEGMENT = "20"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Remark', ORD_REMARK),
('Order_Remark_Qual', 'G'), # General note (not for box label)
])
ord_root.setdefault(ELEMENT, []).append(d)
# == Order Party - WEO30R ==
ELEMENT = "WEO30R"
SEGMENT = "30"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
PART_ID = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'id': None})
PART_EMAIL = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'email': None})
party_root = OrderedDict([
('Segment_Number', SEGMENT),
('Order_Party_Qual', 'CO'), # Consignor - Party sending goods
('Party_External_Ref', PART_ID),
])
ord_root.setdefault(ELEMENT, []).append(party_root)
# == Order Party Address - WEO32R ==
ELEMENT = "WEO32R"
SEGMENT = "32"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
PART_NAME = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'name': None}) or ''
PART_STREET1 = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'street1': None}) or ''
PART_STREET2 = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'street2': None}) or ''
PART_CITY = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'city': None}) or ''
PART_ZIP = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'zip': None}) or ''
PART_COUNTRY = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'country': None}) or ''
PART_STATE = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'state': None}) or ''
PART_PHONE = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'phone': None}) or ''
PART_FAX = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'fax': None}) or ''
PART_LANG = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'language': None}) or ''
PART_VAT = pick.get({'BOTSID': 'pickings'}, {'BOTSID': 'partner', 'vat': None}) or ''
# Spill the first address line over to the second if too long, else truncate
if len(PART_STREET1) > 35 and not PART_STREET2:
space = PART_STREET1.rfind(' ', 0, 35)
if space:
OLD_STREET1 = PART_STREET1
PART_STREET1 = OLD_STREET1[:space]
PART_STREET2 = OLD_STREET1[space+1:]
PART_STREET1 = PART_STREET1[:35]
PART_STREET2 = PART_STREET2[:35]
d = OrderedDict([
('Segment_Number', SEGMENT),
('Company_Name_1', PART_NAME[:35]),
('Address_3', PART_STREET2),
('Postal_Code', PART_ZIP),
('Place_Name', PART_CITY[:35]),
('Country_Code', PART_COUNTRY),
('Language_code', get_dsv_lang(PART_LANG)),
('Telephone_Number', PART_PHONE),
('Telefax_Number', PART_FAX),
('VAT-number', PART_VAT),
('Address_2', PART_STREET1),
('Address_4', PART_STATE),
])
party_root.setdefault(ELEMENT, []).append(d)
plines = pick.getloop({'BOTSID': 'pickings'}, {'BOTSID': 'line'})
for pline in plines:
# == Order Line - WEO40R ==
ELEMENT = "WEO40R"
SEGMENT = "40"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
LINE_ID = pline.get({'BOTSID': 'line', 'id': None})
LINE_SEQ = "%03d" % (int(pline.get({'BOTSID': 'line', 'seq': None})),)
LINE_PRODUCT = pline.get({'BOTSID': 'line', 'product': None}).upper()
LINE_TYPE = '*FIRST'
LINE_QTY = pline.get({'BOTSID': 'line', 'product_qty': None})
LINE_DESC = pline.get({'BOTSID': 'line', 'desc': None})
LINE_VOLUME_NET = pline.get({'BOTSID': 'line', 'volume_net': None})
LINE_WEIGHT = pline.get({'BOTSID': 'line', 'weight': None})
LINE_WEIGHT_NET = pline.get({'BOTSID': 'line', 'weight_net': None})
LINE_PRICE_UNIT = pline.get({'BOTSID': 'line', 'price_unit': None})
LINE_CURRENCY = pline.get({'BOTSID': 'line', 'price_currency': None})
LINE_CUSTOMS_FROM = pline.get({'BOTSID': 'line', 'customs_free_from': None})
LINE_CUSTOMS_FROM = (LINE_CUSTOMS_FROM in ("1", "True", None) and 'FR') or 'T1'
line_root = OrderedDict([
('Segment_Number', SEGMENT),
('Orderline_ID', LINE_SEQ),
('SKU_Reference', LINE_PRODUCT),
('Customs_Status_from', LINE_CUSTOMS_FROM),
('Currency_customs_val', LINE_CURRENCY),
('Type_of_package', LINE_TYPE),
('Measure_unit_specifier', 'PCE'), # FIXME: We only support units UoM currently
('Number_of_packages', LINE_QTY),
('Number_of_units_measure_unit_specifier', LINE_QTY),
('Number_of_units_measure_unit_specifier_per_type_of_package', 1), # FIXME: We only support units UoM currently
('Orderline_remarks', LINE_DESC),
('Net_volume_per_unit_measure_unit_specifier', LINE_VOLUME_NET),
])
ord_root.setdefault(ELEMENT, []).append(line_root)
# == Order Line Customs Value - WEO48R ==
ELEMENT = "WEO48R"
SEGMENT = "48"
rec_counts[SEGMENT] = rec_counts.setdefault(SEGMENT, 0) + 1
d = OrderedDict([
('Segment_Number', SEGMENT),
('Customs_value_qual', '001'), # Goods value
('Customs_value', "%015.02f" % (float(LINE_PRICE_UNIT) * float(LINE_QTY)),),
('Positive-sign', 'Y'),
])
line_root.setdefault(ELEMENT, []).append(d)
# Catch all
else:
raise NotImplementedError('Unable to handle order with state %s' % (ORD_STATE,))
# == FOOTER - WEO99R ==
rec_counts['99'] = len(rec_counts.keys())
for FOOT_REC_TYPE, FOOT_REC_COUNT in rec_counts.iteritems():
ELEMENT = "WEO99R"
SEGMENT = "99"
d = OrderedDict([
('Segment_Number', SEGMENT),
('RecordType', FOOT_REC_TYPE),
('Total_Nbrs_of_Rec', FOOT_REC_COUNT),
])
root.setdefault(ELEMENT, []).append(d)
root = OrderedDict(sorted(root.items(), key=lambda t: t[0]))
document = OrderedDict([('WEO', root)])
data = serialize_dsv_xml(document, SCHEMA)
out.root = data
return
def insert_user(user_data_orig):
user_data = dict(user_data_orig)
user_data['password'] = util.get_hashed_password(user_data['password'])
user_data.update({'reg_date': util.get_datetime()})
insert.new_user(user_data)
elif len(sys.argv) > 3:
for i in range(len(sys.argv) - 2): #判断命令格式是否正确
if not "=" in sys.argv[i + 2] or not "-D" in sys.argv[
i + 2]: #如果没有=或者-D,
print("Usage python %s [stream_name] -Dxxx=xxx" %
sys.argv[0]) #sys.argv[0]:"scan.py"
sys.exit()
for i in range(len(sys.argv) - 2):
tmp = sys.argv[i + 2].split("=", 1) #以=为分隔符，分隔一次成两个
if tmp[0].__eq__("-DSCM_TYPE") and tmp[1].__eq__(
"code_tfs_git"): #将参数-DSCM为"code_tfs_git":
offline_properties_info[tmp[0].replace("-D", "")] = "git"
else:
offline_properties_info[tmp[0].replace(
"-D", "")] = tmp[1].replace("\n", "")
main_input(sys.argv[1])
else:
print(util.get_datetime() +
" Usage python %s [stream_name]_[cpplint&astyle&pylint] " %
sys.argv[0])
sys.exit()
except Exception as e:
params_sub['startTime'] = '0'
params_sub['endTime'] = str(int(time.time()) / 1000)
params_sub['flag'] = 2
params_sub['msg'] = 'Exception Error: ' + str(e)
codecc_web.codecc_upload_task_log(config.params_root, params_sub)
traceback.print_exc()
sys.exit(1)
